Commit 922ea4b5 authored by Kevin's avatar Kevin

Merge branch 'refactoring2' into 'master'

Merge refactoring2

See merge request !1
parents 640ce593 f3279186
Pipeline #25779803 passed with stage
in 1 minute and 31 seconds
......@@ -12,3 +12,4 @@ run.sh
onionr/data-encrypted.dat
onionr/.onionr-lock
core
.vscode/*
test:
script:
- apt-get update -qy
- apt-get install -y python3-dev python3-pip tor
- pip3 install -r requirements.txt
- make test
\ No newline at end of file
......@@ -20,7 +20,7 @@ test:
@sleep 1
@rm -rf onionr/data-backup
@mv onionr/data onionr/data-backup | true > /dev/null 2>&1
-@cd onionr; ./tests.py; ./cryptotests.py;
-@cd onionr; ./tests.py;
@rm -rf onionr/data
@mv onionr/data-backup onionr/data | true > /dev/null 2>&1
......
......@@ -22,16 +22,17 @@ from flask import request, Response, abort
from multiprocessing import Process
from gevent.wsgi import WSGIServer
import sys, random, threading, hmac, hashlib, base64, time, math, os, logger, config
from core import Core
from onionrblockapi import Block
import onionrutils, onionrcrypto
class API:
'''
Main HTTP API (Flask)
'''
def validateToken(self, token):
'''
Validate that the client token (hmac) matches the given token
Validate that the client token matches the given token
'''
try:
if not hmac.compare_digest(self.clientToken, token):
......@@ -50,8 +51,8 @@ class API:
'''
config.reload()
if config.get('devmode', True):
if config.get('dev_mode', True):
self._developmentMode = True
logger.set_level(logger.LEVEL_DEBUG)
else:
......@@ -64,29 +65,26 @@ class API:
self._crypto = onionrcrypto.OnionrCrypto(self._core)
self._utils = onionrutils.OnionrUtils(self._core)
app = flask.Flask(__name__)
bindPort = int(config.get('client')['port'])
bindPort = int(config.get('client.port', 59496))
self.bindPort = bindPort
self.clientToken = config.get('client')['client_hmac']
self.clientToken = config.get('client.hmac')
self.timeBypassToken = base64.b16encode(os.urandom(32)).decode()
self.i2pEnabled = config.get('i2p')['host']
self.i2pEnabled = config.get('i2p.host', False)
self.mimeType = 'text/plain'
with open('data/time-bypass.txt', 'w') as bypass:
bypass.write(self.timeBypassToken)
if not os.environ.get("WERKZEUG_RUN_MAIN") == "true":
logger.debug('Your web password (KEEP SECRET): ' + logger.colors.underline + self.clientToken)
if not debug and not self._developmentMode:
hostNums = [random.randint(1, 255), random.randint(1, 255), random.randint(1, 255)]
self.host = '127.' + str(hostNums[0]) + '.' + str(hostNums[1]) + '.' + str(hostNums[2])
hostOctets = [127, random.randint(0x02, 0xFF), random.randint(0x02, 0xFF), random.randint(0x02, 0xFF)]
self.host = '.'.join(hostOctets)
else:
self.host = '127.0.0.1'
hostFile = open('data/host.txt', 'w')
hostFile.write(self.host)
hostFile.close()
with open('data/host.txt', 'w') as file:
file.write(self.host)
@app.before_request
def beforeReq():
......@@ -127,7 +125,7 @@ class API:
except:
data = ''
startTime = math.floor(time.time())
# we should keep a hash DB of requests (with hmac) to prevent replays
action = request.args.get('action')
#if not self.debug:
token = request.args.get('token')
......@@ -192,8 +190,6 @@ class API:
pass
elif action == 'ping':
resp = Response("pong!")
elif action == 'getHMAC':
resp = Response(self._crypto.generateSymmetric())
elif action == 'getSymmetric':
resp = Response(self._crypto.generateSymmetric())
elif action == 'getDBHash':
......@@ -213,13 +209,12 @@ class API:
resp = Response('')
# setData should be something the communicator initiates, not this api
elif action == 'getData':
resp = ''
if self._utils.validateHash(data):
if not os.path.exists('data/blocks/' + data + '.db'):
try:
resp = base64.b64encode(self._core.getData(data))
except TypeError:
resp = ""
if resp == False:
block = Block(hash=data.encode(), core=self._core)
resp = base64.b64encode(block.getRaw().encode()).decode()
if len(resp) == 0:
abort(404)
resp = ""
resp = Response(resp)
......@@ -258,7 +253,7 @@ class API:
return resp
if not os.environ.get("WERKZEUG_RUN_MAIN") == "true":
logger.info('Starting client on ' + self.host + ':' + str(bindPort) + '...', timestamp=True)
logger.info('Starting client on ' + self.host + ':' + str(bindPort) + '...', timestamp=False)
try:
self.http_server = WSGIServer((self.host, bindPort), app)
......
This diff is collapsed.
This diff is collapsed.
......@@ -28,9 +28,20 @@ def get(key, default = None):
Gets the key from configuration, or returns `default`
'''
if is_set(key):
return get_config()[key]
return default
key = str(key).split('.')
data = _config
last = key.pop()
for item in key:
if (not item in data) or (not type(data[item]) == dict):
return default
data = data[item]
if not last in data:
return default
return data[last]
def set(key, value = None, savefile = False):
'''
......@@ -38,16 +49,40 @@ def set(key, value = None, savefile = False):
'''
global _config
key = str(key).split('.')
data = _config
last = key.pop()
for item in key:
if (not item in data) or (not type(data[item]) == dict):
data[item] = dict()
data = data[item]
if value is None:
del _config[key]
del data[last]
else:
_config[key] = value
data[last] = value
if savefile:
save()
def is_set(key):
return key in get_config() and not get_config()[key] is None
key = str(key).split('.')
data = _config
last = key.pop()
for item in key:
if (not item in data) or (not type(data[item]) == dict):
return False
data = data[item]
if not last in data:
return False
return True
def check():
'''
......@@ -71,7 +106,7 @@ def save():
check()
try:
with open(get_config_file(), 'w', encoding="utf8") as configfile:
json.dump(get_config(), configfile, indent=2, sort_keys=True)
json.dump(get_config(), configfile, indent=2)
except:
logger.warn('Failed to write to configuration file.')
......
This diff is collapsed.
This diff is collapsed.
......@@ -123,18 +123,18 @@ def get_file():
return _outputfile
def raw(data):
def raw(data, fd = sys.stdout):
'''
Outputs raw data to console without formatting
'''
if get_settings() & OUTPUT_TO_CONSOLE:
print(data)
ts = fd.write('%s\n' % data)
if get_settings() & OUTPUT_TO_FILE:
with open(_outputfile, "a+") as f:
f.write(colors.filter(data) + '\n')
def log(prefix, data, color = '', timestamp=True):
def log(prefix, data, color = '', timestamp=True, fd = sys.stdout):
'''
Logs the data
prefix : The prefix to the output
......@@ -149,7 +149,7 @@ def log(prefix, data, color = '', timestamp=True):
if not get_settings() & USE_ANSI:
output = colors.filter(output)
raw(output)
raw(output, fd = fd)
def readline(message = ''):
'''
......@@ -218,14 +218,14 @@ def warn(data, timestamp=True):
# error: when only one function, module, or process of the program encountered a problem and must stop
def error(data, error=None, timestamp=True):
if get_level() <= LEVEL_ERROR:
log('-', data, colors.fg.red, timestamp=timestamp)
log('-', data, colors.fg.red, timestamp=timestamp, fd = sys.stderr)
if not error is None:
debug('Error: ' + str(error) + parse_error())
# fatal: when the something so bad has happened that the program must stop
def fatal(data, timestamp=True):
if get_level() <= LEVEL_FATAL:
log('#', data, colors.bg.red + colors.fg.green + colors.bold, timestamp=timestamp)
log('#', data, colors.bg.red + colors.fg.green + colors.bold, timestamp=timestamp, fd = sys.stderr)
# returns a formatted error message
def parse_error():
......
......@@ -19,6 +19,7 @@
'''
import subprocess, os, random, sys, logger, time, signal
from onionrblockapi import Block
class NetController:
'''
......@@ -102,7 +103,7 @@ DataDirectory data/tordata/
logger.fatal("Got keyboard interrupt")
return False
logger.info('Finished starting Tor', timestamp=True)
logger.debug('Finished starting Tor.', timestamp=True)
self.readyState = True
myID = open('data/hs/hostname', 'r')
......
This diff is collapsed.
This diff is collapsed.
......@@ -17,7 +17,13 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import nacl.signing, nacl.encoding, nacl.public, nacl.hash, nacl.secret, os, binascii, base64, hashlib, logger, onionrproofs, time, math
import nacl.signing, nacl.encoding, nacl.public, nacl.hash, nacl.secret, os, binascii, base64, hashlib, logger, onionrproofs, time, math, sys
# secrets module was added into standard lib in 3.6+
if sys.version_info[0] == 3 and sys.version_info[1] < 6:
from dependencies import secrets
elif sys.version_info[0] == 3 and sys.version_info[1] >= 6:
import secrets
class OnionrCrypto:
def __init__(self, coreInstance):
......@@ -27,6 +33,8 @@ class OnionrCrypto:
self.pubKey = None
self.privKey = None
self.secrets = secrets
self.pubKeyPowToken = None
#self.pubKeyPowHash = None
......@@ -51,7 +59,7 @@ class OnionrCrypto:
with open(self._keyFile, 'w') as keyfile:
keyfile.write(self.pubKey + ',' + self.privKey)
with open(self.keyPowFile, 'w') as keyPowFile:
proof = onionrproofs.POW(self.pubKey)
proof = onionrproofs.DataPOW(self.pubKey)
logger.info('Doing necessary work to insert our public key')
while True:
time.sleep(0.2)
......@@ -102,7 +110,7 @@ class OnionrCrypto:
retData = key.sign(data).signature
return retData
def pubKeyEncrypt(self, data, pubkey, anonymous=False, encodedData=False):
def pubKeyEncrypt(self, data, pubkey, anonymous=True, encodedData=False):
'''Encrypt to a public key (Curve25519, taken from base32 Ed25519 pubkey)'''
retVal = ''
......@@ -241,35 +249,34 @@ class OnionrCrypto:
pass
return nacl.hash.blake2b(data)
def verifyPow(self, blockContent, metadata):
def verifyPow(self, blockContent):
'''
Verifies the proof of work associated with a block
'''
retData = False
if not (('powToken' in metadata) and ('powHash' in metadata)):
return False
dataLen = len(blockContent)
expectedHash = self.blake2bHash(base64.b64decode(metadata['powToken']) + self.blake2bHash(blockContent.encode()))
difficulty = 0
try:
expectedHash = expectedHash.decode()
blockContent = blockContent.encode()
except AttributeError:
pass
if metadata['powHash'] == expectedHash:
difficulty = math.floor(dataLen / 1000000)
mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode()
puzzle = mainHash[:difficulty]
blockHash = self.sha3Hash(blockContent)
try:
blockHash = blockHash.decode() # bytes on some versions for some reason
except AttributeError:
pass
difficulty = math.floor(dataLen / 1000000)
mainHash = '0000000000000000000000000000000000000000000000000000000000000000'#nacl.hash.blake2b(nacl.utils.random()).decode()
puzzle = mainHash[:difficulty]
if metadata['powHash'][:difficulty] == puzzle:
# logger.debug('Validated block pow')
retData = True
else:
logger.debug("Invalid token (#1)")
if blockHash[:difficulty] == puzzle:
# logger.debug('Validated block pow')
retData = True
else:
logger.debug('Invalid token (#2): Expected hash %s, got hash %s...' % (metadata['powHash'], expectedHash))
logger.debug("Invalid token, bad proof")
return retData
'''
Onionr - P2P Microblogging Platform & Social network. Run with 'help' for usage.
Onionr - P2P Microblogging Platform & Social network.
This file contains exceptions for onionr
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
......@@ -9,42 +13,37 @@
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import hmac, base64, time, math
class TimedHMAC:
def __init__(self, base64Key, data, hashAlgo):
'''
base64Key = base64 encoded key
data = data to hash
expire = time expiry in epoch
hashAlgo = string in hashlib.algorithms_available
Maximum of 10 seconds grace period
'''
self.data = data
self.expire = math.floor(time.time())
self.hashAlgo = hashAlgo
self.b64Key = base64Key
generatedHMAC = hmac.HMAC(base64.b64decode(base64Key).decode(), digestmod=self.hashAlgo)
generatedHMAC.update(data + expire)
self.HMACResult = generatedHMAC.hexdigest()
return
def check(self, data):
'''
Check a hash (and verify time is sane)
'''
testHash = hmac.HMAC(base64.b64decode(base64Key).decode(), digestmod=self.hashAlgo)
testHash.update(data + math.floor(time.time()))
testHash = testHash.hexdigest()
if hmac.compare_digest(testHash, self.HMACResult):
return true
return false
# general exceptions
class NotFound(Exception):
pass
class Unknown(Exception):
pass
class Invalid(Exception):
pass
# communicator exceptions
class OnlinePeerNeeded(Exception):
pass
# crypto exceptions
class InvalidPubkey(Exception):
pass
# block exceptions
class InvalidMetadata(Exception):
pass
class InvalidHexHash(Exception):
'''When a string is not a valid hex string of appropriate length for a hash value'''
pass
# network level exceptions
class MissingPort(Exception):
pass
class InvalidAddress(Exception):
pass
'''
Onionr - P2P Microblogging Platform & Social network.
This file contains both the OnionrCommunicate class for communcating with peers
'''
'''
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
\ No newline at end of file
......@@ -63,21 +63,21 @@ def enable(name, onionr = None, start_event = True):
if exists(name):
enabled_plugins = get_enabled_plugins()
if not name in enabled_plugins:
enabled_plugins.append(name)
config_plugins = config.get('plugins')
config_plugins['enabled'] = enabled_plugins
config.set('plugins', config_plugins, True)
events.call(get_plugin(name), 'enable', onionr)
if start_event is True:
start(name)
return True
try:
events.call(get_plugin(name), 'enable', onionr)
except ImportError: # Was getting import error on Gitlab CI test "data"
return False
else:
enabled_plugins.append(name)
config.set('plugins.enabled', enabled_plugins, True)
if start_event is True:
start(name)
return True
else:
return False
else:
logger.error('Failed to enable plugin \"' + name + '\", disabling plugin.')
logger.error('Failed to enable plugin \"%s\", disabling plugin.' % name)
disable(name)
return False
......@@ -93,9 +93,7 @@ def disable(name, onionr = None, stop_event = True):
if is_enabled(name):
enabled_plugins = get_enabled_plugins()
enabled_plugins.remove(name)
config_plugins = config.get('plugins')
config_plugins['enabled'] = enabled_plugins
config.set('plugins', config_plugins, True)
config.set('plugins.enabled', enabled_plugins, True)
if exists(name):
events.call(get_plugin(name), 'disable', onionr)
......@@ -121,9 +119,9 @@ def start(name, onionr = None):
return plugin
except:
logger.error('Failed to start module \"' + name + '\".')
logger.error('Failed to start module \"%s\".' % name)
else:
logger.error('Failed to start nonexistant module \"' + name + '\".')
logger.error('Failed to start nonexistant module \"%s\".' % name)
return None
......@@ -145,9 +143,9 @@ def stop(name, onionr = None):
return plugin
except:
logger.error('Failed to stop module \"' + name + '\".')
logger.error('Failed to stop module \"%s\".' % name)
else:
logger.error('Failed to stop nonexistant module \"' + name + '\".')
logger.error('Failed to stop nonexistant module \"%s\".' % name)
return None
......@@ -187,7 +185,7 @@ def get_enabled_plugins():
config.reload()
return config.get('plugins')['enabled']
return config.get('plugins.enabled', list())
def is_enabled(name):
'''
......
......@@ -18,11 +18,41 @@
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, logger, sys, base64
import nacl.encoding, nacl.hash, nacl.utils, time, math, threading, binascii, logger, sys, base64, json
import core
class POW:
def pow(self, reporting = False):
class DataPOW:
def __init__(self, data, threadCount = 5):
self.foundHash = False
self.difficulty = 0
self.data = data
self.threadCount = threadCount
dataLen = sys.getsizeof(data)
self.difficulty = math.floor(dataLen / 1000000)
if self.difficulty <= 2:
self.difficulty = 4
try:
self.data = self.data.encode()
except AttributeError:
pass
self.data = nacl.hash.blake2b(self.data)
logger.info('Computing POW (difficulty: %s)...' % self.difficulty)
self.mainHash = '0' * 70
self.puzzle = self.mainHash[0:min(self.difficulty, len(self.mainHash))]
myCore = core.Core()
for i in range(max(1, threadCount)):
t = threading.Thread(name = 'thread%s' % i, target = self.pow, args = (True,myCore))
t.start()
return
def pow(self, reporting = False, myCore = None):
startTime = math.floor(time.time())
self.hashing = True
self.reporting = reporting
......@@ -30,7 +60,7 @@ class POW:
answer = ''
heartbeat = 200000
hbCount = 0
myCore = core.Core()
while self.hashing:
rand = nacl.utils.random()
token = nacl.hash.blake2b(rand + self.data).decode()
......@@ -39,23 +69,61 @@ class POW:
self.hashing = False
iFound = True
break
else:
logger.debug('POW thread exiting, another thread found result')
if iFound:
endTime = math.floor(time.time())
if self.reporting:
logger.info('Found token ' + token, timestamp=True)
logger.info('rand value: ' + base64.b64encode(rand).decode())
logger.info('took ' + str(endTime - startTime) + ' seconds', timestamp=True)
logger.debug('Found token after %s seconds: %s' % (endTime - startTime, token), timestamp=True)
logger.debug('Random value was: %s' % base64.b64encode(rand).decode())
self.result = (token, rand)
def __init__(self, data):
def shutdown(self):
self.hashing = False
self.puzzle = ''
def changeDifficulty(self, newDiff):
self.difficulty = newDiff
def getResult(self):
'''
Returns the result then sets to false, useful to automatically clear the result
'''
try:
retVal = self.result
except AttributeError:
retVal = False
self.result = False
return retVal
def waitForResult(self):
'''
Returns the result only when it has been found, False if not running and not found
'''
result = False
try:
while True:
result = self.getResult()