Commit 3ea9d674 authored by Kevin Froman's avatar Kevin Froman

added sneakernet auto importing

parent 160469f5
Pipeline #130937788 failed with stage
in 8 minutes and 2 seconds
......@@ -46,6 +46,13 @@
"command": "scripts/disable-dev-config.py",
"group": "dev",
"problemMatcher": []
},
{
"label": "Format default config",
"type": "process",
"command": "scripts/pretty-default-config.py",
"group": "dev",
"problemMatcher": []
}
]
}
\ No newline at end of file
......@@ -8,9 +8,9 @@ stem==1.8.0
deadsimplekv==0.3.1
unpaddedbase32==0.2.0
streamedrequests==1.0.0
jinja2==2.11.1
toomanyobjs==1.1.0
niceware==0.2.1
psutil==5.7.0
filenuke==0.0.0
mimcvdf==1.0.0
watchdog==0.10.2
\ No newline at end of file
......@@ -115,7 +115,7 @@ itsdangerous==1.1.0 \
jinja2==2.11.1 \
--hash=sha256:93187ffbc7808079673ef52771baa950426fd664d3aad1d0fa3e95644360e250 \
--hash=sha256:b0eaf100007721b5c16c1fc1eecb87409464edc10469ddc9a22a27a99123be49 \
# via -r requirements.in, flask
# via flask
markupsafe==1.1.1 \
--hash=sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473 \
--hash=sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161 \
......@@ -154,6 +154,9 @@ niceware==0.2.1 \
--hash=sha256:0f8b192f2a1e800e068474f6e208be9c7e2857664b33a96f4045340de4e5c69c \
--hash=sha256:cf2dc0e1567d36d067c61b32fed0f1b9c4534ed511f9eeead4ba548d03b5c9eb \
# via -r requirements.in
pathtools==0.1.2 \
--hash=sha256:7c35c5421a39bb82e58018febd90e3b6e5db34c5443aaaf742b3f33d4655f1c0 \
# via watchdog
psutil==5.7.0 \
--hash=sha256:1413f4158eb50e110777c4f15d7c759521703bd6beb58926f1d562da40180058 \
--hash=sha256:298af2f14b635c3c7118fd9183843f4e73e681bb6f01e12284d4d70d48a60953 \
......@@ -221,6 +224,9 @@ urllib3==1.25.8 \
--hash=sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc \
--hash=sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc \
# via -r requirements.in, requests
watchdog==0.10.2 \
--hash=sha256:c560efb643faed5ef28784b2245cf8874f939569717a4a12826a173ac644456b \
# via -r requirements.in
werkzeug==0.15.5 \
--hash=sha256:87ae4e5b5366da2347eb3116c0e6c681a0e939a33b2805e2c0cbd282664932c4 \
--hash=sha256:a13b74dd3c45f758d4ebdb224be8f1ab8ef58b3c0ffc1783a8c7d9f4f50227e6 \
......
......@@ -23,7 +23,7 @@ conf['general']['max_block_age'] = 2678400
conf['log']['file']['remove_on_exit'] = True
conf['transports']['lan'] = True
conf['transports']['tor'] = True
conf['transports']['manual_disk'] = True
conf['transports']['sneakernet'] = True
json.dump(conf, open('static-data/default_config.json', 'w'), sort_keys=True, indent=4)
......@@ -47,7 +47,7 @@ WSGI_SERVER_REQUEST_TIMEOUT_SECS = 120
MAX_NEW_PEER_QUEUE = 1000
BLOCK_EXPORT_FILE_EXT = '.dat'
BLOCK_EXPORT_FILE_EXT = '.onionr'
# Begin OnionrValues migrated values
......
......@@ -37,6 +37,7 @@ from .killdaemon import kill_daemon # noqa
from utils.boxprint import bordered
from lan import LANManager
from lan.server import LANServer
from sneakernet import sneakernet_import_thread
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
......@@ -169,6 +170,8 @@ def daemon():
Thread(target=LANServer(shared_state).start_server,
daemon=True).start()
LANManager(shared_state).start()
if config.get('transports.sneakernet', True):
Thread(target=sneakernet_import_thread, daemon=True).start()
communicator.startCommunicator(shared_state)
clean_ephemeral_services()
......
......@@ -8,6 +8,7 @@ import logger
import onionrstorage
from utils import createdirs
from onionrutils import stringvalidators
from etc.onionrvalues import BLOCK_EXPORT_FILE_EXT
import filepaths
import os
......@@ -31,8 +32,8 @@ from coredb import blockmetadb
def _do_export(b_hash):
createdirs.create_dirs()
data = onionrstorage.getData(b_hash)
with open('%s/%s.dat' % (filepaths.export_location,
b_hash), 'wb') as export:
with open('%s/%s%s' % (filepaths.export_location,
b_hash, BLOCK_EXPORT_FILE_EXT), 'wb') as export:
export.write(data)
logger.info('Block exported as file', terminal=True)
......
......@@ -23,6 +23,7 @@ from onionrutils import blockmetadata
from coredb import blockmetadb
import filepaths
import onionrcrypto as crypto
from etc.onionrvalues import BLOCK_EXPORT_FILE_EXT
def import_new_blocks(scanDir=''):
'''
This function is intended to scan for new blocks ON THE DISK and import them
......@@ -33,19 +34,19 @@ def import_new_blocks(scanDir=''):
scanDir = filepaths.block_data_location
if not scanDir.endswith('/'):
scanDir += '/'
for block in glob.glob(scanDir + "*.dat"):
if block.replace(scanDir, '').replace('.dat', '') not in blockList:
for block in glob.glob(scanDir + "*%s" % (BLOCK_EXPORT_FILE_EXT,)):
if block.replace(scanDir, '').replace(BLOCK_EXPORT_FILE_EXT, '') not in blockList:
exist = True
logger.info('Found new block on dist %s' % block, terminal=True)
with open(block, 'rb') as newBlock:
block = block.replace(scanDir, '').replace('.dat', '')
if crypto.hashers.sha3_hash(newBlock.read()) == block.replace('.dat', ''):
blockmetadb.add_to_block_DB(block.replace('.dat', ''), dataSaved=True)
logger.info('Imported block %s.' % block, terminal=True)
block = block.replace(scanDir, '').replace(BLOCK_EXPORT_FILE_EXT, '')
if crypto.hashers.sha3_hash(newBlock.read()) == block.replace(BLOCK_EXPORT_FILE_EXT, ''):
blockmetadb.add_to_block_DB(block.replace(BLOCK_EXPORT_FILE_EXT, ''), dataSaved=True)
logger.info('Imported block %s' % block, terminal=True)
blockmetadata.process_block_metadata(block)
else:
logger.warn('Failed to verify hash for %s' % block, terminal=True)
if not exist:
logger.info('No blocks found to import', terminal=True)
import_new_blocks.onionr_help = f"Scans the Onionr data directory under {filepaths.block_data_location} for new block files (.dat, .db not supported) to import"
import_new_blocks.onionr_help = f"Scans the Onionr data directory under {filepaths.block_data_location} for new block files (.db not supported) to import"
'''
Onionr - Private P2P Communication
"""Onionr - Private P2P Communication.
This processes metadata for Onionr blocks
'''
'''
Detect new block files in a given directory
"""
import os
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
import config
from filepaths import block_data_location
from etc.onionrvalues import BLOCK_EXPORT_FILE_EXT
from onionrblocks.blockimporter import import_block_from_data
"""
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
......@@ -16,44 +24,31 @@
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
'''
# useful libraries
import logger, config
import os, sys, json, time, random, shutil, base64, getpass, datetime, re
import onionrusers, onionrexceptions
from onionrutils import stringvalidators
plugin_name = 'metadataprocessor'
# event listeners
def _processForwardKey(api, myBlock):
'''
Get the forward secrecy key specified by the user for us to use
'''
peer = onionrusers.OnionrUser(myBlock.signer)
key = myBlock.getMetadata('newFSKey')
# We don't need to validate here probably, but it helps
if stringvalidators.validate_pub_key(key):
peer.addForwardKey(key)
else:
raise onionrexceptions.InvalidPubkey("%s is not a valid pubkey key" % (key,))
def on_processblocks(api, data=None):
# Generally fired by utils.
myBlock = api.data['block']
blockType = api.data['type']
# Process specific block types
# forwardKey blocks, add a new forward secrecy key for a peer
if blockType == 'forwardKey':
if api.data['validSig'] == True:
_processForwardKey(api, myBlock)
def on_init(api, data = None):
pluginapi = api
return
"""
watch_paths = config.get('transports.sneakernet.paths', list([]))
if block_data_location not in watch_paths:
watch_paths.append(block_data_location)
class _Importer(FileSystemEventHandler):
@staticmethod
def on_created(event):
if not event.src_path.endswith(BLOCK_EXPORT_FILE_EXT):
return
with open(event.src_path, 'rb') as block_file:
import_block_from_data(block_file.read())
if block_data_location in event.src_path:
try:
os.remove(event.src_path)
except FileNotFoundError:
pass
def sneakernet_import_thread():
observer = Observer()
for path in watch_paths:
observer.schedule(_Importer(), path, recursive=True)
observer.start()
while observer.isAlive():
observer.join(60)
\ No newline at end of file
{
"name" : "metadataprocessor",
"version" : "1.0",
"author" : "onionr"
}
......@@ -58,7 +58,7 @@ class OnionrConfig(unittest.TestCase):
self.assertEqual(conf['tor']['use_bridge'], False)
self.assertEqual(conf['tor']['use_existing_tor'], False)
self.assertEqual(conf['transports']['lan'], True)
self.assertEqual(conf['transports']['manual_disk'], True)
self.assertEqual(conf['transports']['sneakernet'], True)
self.assertEqual(conf['transports']['tor'], True)
self.assertEqual(conf['ui']['theme'], 'dark')
unittest.main()
......@@ -19,6 +19,6 @@ class TestOnionrValues(unittest.TestCase):
self.assertEqual(onionrvalues.MAX_BLOCK_CLOCK_SKEW, 120)
def test_block_export_ext(self):
self.assertEqual(onionrvalues.BLOCK_EXPORT_FILE_EXT, '.dat')
self.assertEqual(onionrvalues.BLOCK_EXPORT_FILE_EXT, '.onionr')
unittest.main()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment