Commit 3fc09c26 authored by Christopher Ronning's avatar Christopher Ronning

Merge branch '28-data-download' into 'master'

Real Okay Design for Torrent Downloader

See merge request !12
parents b7a05d55 289a85da
Pipeline #7353767 passed with stages
in 1 minute and 49 seconds
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 15
VisualStudioVersion = 15.0.26114.2
VisualStudioVersion = 15.0.26228.4
MinimumVisualStudioVersion = 10.0.40219.1
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{3D662F2C-304E-4F35-B7BE-AAE44DE4EB04}"
EndProject
......@@ -20,21 +19,29 @@ EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
Debug|x64 = Debug|x64
Debug|x86 = Debug|x86
Release|Any CPU = Release|Any CPU
Release|x64 = Release|x64
Release|x86 = Release|x86
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{48964819-E837-49C3-96C1-8F365D5B2CA2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{48964819-E837-49C3-96C1-8F365D5B2CA2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{48964819-E837-49C3-96C1-8F365D5B2CA2}.Debug|x64.ActiveCfg = Debug|Any CPU
{48964819-E837-49C3-96C1-8F365D5B2CA2}.Debug|x86.ActiveCfg = Debug|Any CPU
{48964819-E837-49C3-96C1-8F365D5B2CA2}.Release|Any CPU.ActiveCfg = Release|Any CPU
{48964819-E837-49C3-96C1-8F365D5B2CA2}.Release|Any CPU.Build.0 = Release|Any CPU
{48964819-E837-49C3-96C1-8F365D5B2CA2}.Release|x64.ActiveCfg = Release|Any CPU
{48964819-E837-49C3-96C1-8F365D5B2CA2}.Release|x86.ActiveCfg = Release|Any CPU
{D23B4D6A-2366-46E7-9323-4552F1BF4A1A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D23B4D6A-2366-46E7-9323-4552F1BF4A1A}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D23B4D6A-2366-46E7-9323-4552F1BF4A1A}.Debug|x64.ActiveCfg = Debug|Any CPU
{D23B4D6A-2366-46E7-9323-4552F1BF4A1A}.Debug|x86.ActiveCfg = Debug|Any CPU
{D23B4D6A-2366-46E7-9323-4552F1BF4A1A}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D23B4D6A-2366-46E7-9323-4552F1BF4A1A}.Release|Any CPU.Build.0 = Release|Any CPU
{D23B4D6A-2366-46E7-9323-4552F1BF4A1A}.Release|x64.ActiveCfg = Release|Any CPU
{D23B4D6A-2366-46E7-9323-4552F1BF4A1A}.Release|x86.ActiveCfg = Release|Any CPU
{D93126C1-A49F-4B4A-8F67-E9261E65FD9A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D93126C1-A49F-4B4A-8F67-E9261E65FD9A}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D93126C1-A49F-4B4A-8F67-E9261E65FD9A}.Debug|x64.ActiveCfg = Debug|x64
......
No preview for this file type
......@@ -8,12 +8,13 @@ using Ditto.Common;
namespace Ditto.BitTorrent
{
class Client : IDisposable {
// Torrents we expect to be loaded into our test peer.
readonly byte[] veryTinyKnownInfohash = "ea45080eab61ab465f647e6366f775bf25f69a61".FromHex();
readonly byte[] lessTinyKnownInfohash = "68d22f0f856ca5056e009ac53597a66c0cb03068".FromHex();
// Torrents we do not expect to be loaded in our test peer, but which should have many peers online.
readonly byte[] ubuntuUnknownInfohash = "34930674ef3bb9317fb5f263cca830f52685235b".FromHex();
class Client : IDisposable
{
readonly private IPAddress myIpAddress;
public static readonly Int32 myPort = 6881;
public static readonly byte[] peerId = new byte[20].FillRandom();
public static bool extensionsEnabled = true; // Extensions enabled by default -- option to disable?
private DHT.Client dht;
......@@ -21,39 +22,58 @@ namespace Ditto.BitTorrent
public Client() {
dht = new DHT.Client();
myIpAddress = IPAddress.Any;
"ditto.to#".ToASCII().CopyTo(peerId, 0);
}
public async Task Example(IPAddress[] bootstrapAddresses) {
foreach (var address in bootstrapAddresses) {
var bootstrapNode = new IPEndPoint(address, 9527);
dht.AddNode(bootstrapNode);
}
var ubuntuPeers = await dht.GetPeers(ubuntuUnknownInfohash);
public async Task Example(IPAddress[] bootstrapAddresses, IPEndPoint peer=null)
{
if (peer == null)
{
logger.LogInformation(LoggingEvents.DHT_PROTOCOL_MSG,
$"Requested peers for Ubuntu {ubuntuUnknownInfohash.ToHex()} and got {ubuntuPeers.Count}!");
foreach (var address in bootstrapAddresses)
{
var bootstrapNode = new IPEndPoint(address, 9527);
dht.AddNode(bootstrapNode);
}
foreach (var ep in ubuntuPeers)
var ubuntuPeers = await dht.GetPeers(KnownTorrents.ubuntuUnknownInfohash);
{
logger.LogInformation(LoggingEvents.ATTEMPT_CONNECTION, $"Attempting to connect to peer at {ep}.");
logger.LogInformation(LoggingEvents.DHT_PROTOCOL_MSG,
$"Requested peers for Ubuntu {KnownTorrents.ubuntuUnknownInfohash.ToHex()} and got {ubuntuPeers.Count}!");
try
{
var connection = new Ditto.PeerProtocol.PeerConnection(IPAddress.Any);
connection.InitiateHandshake(ep.Address, ep.Port, ubuntuUnknownInfohash);
break;
}
catch (Exception ex)
foreach (var ep in ubuntuPeers)
{
logger.LogError(LoggingEvents.DHT_ERROR, "It failed: " + ex);
await Task.Delay(1000);
logger.LogError(LoggingEvents.DHT_ERROR, "Do I have another peer to try?");
continue;
logger.LogInformation(LoggingEvents.ATTEMPT_CONNECTION, $"Attempting to connect to peer at {ep}.");
try
{
var testTorrent = new Ditto.PeerProtocol.TorrentManager(KnownTorrents.ubuntuUnknownInfohash);
testTorrent.AddPeer(peer); // this will need to be managed more systematically
break;
}
catch (Exception ex)
{
logger.LogError(LoggingEvents.DHT_ERROR, "It failed: " + ex);
await Task.Delay(1000);
logger.LogError(LoggingEvents.DHT_ERROR, "Do I have another peer to try?");
continue;
}
}
}
logger.LogInformation("Done.");
logger.LogInformation("Done.");
}
}
else
{
try
{
var ubuntuTorrent = new Ditto.PeerProtocol.TorrentManager(KnownTorrents.netBSDInfohash);
ubuntuTorrent.AddPeer(peer); // this will need to be managed more systematically
}
catch (Exception ex)
{
logger.LogError(LoggingEvents.DHT_ERROR, "It failed: " + ex);
}
}
}
......
using Ditto.Common;
namespace Ditto.BitTorrent
{
class KnownTorrents
{
// Torrents we expect to be loaded into our test peer.
public static readonly byte[] veryTinyKnownInfohash = "ea45080eab61ab465f647e6366f775bf25f69a61".FromHex();
public static readonly byte[] lessTinyKnownInfohash = "68d22f0f856ca5056e009ac53597a66c0cb03068".FromHex();
// Torrents we do not expect to be loaded in our test peer, but which should have many peers online.
public static readonly byte[] ubuntuUnknownInfohash = "34930674ef3bb9317fb5f263cca830f52685235b".FromHex();
public static readonly byte[] netBSDInfohash = "a04028df0f94de0db51f3132dec47c754ffc20b1".FromHex();
}
}
......@@ -14,9 +14,13 @@ namespace Ditto {
static ILogger logger { get; } = GlobalLogger.CreateLogger<CLInterface>();
public static int Main(string[] args)
public static int RunCLI(string[] args)
{
var cli = new CommandLineApplication {
IPAddress peerIP;
IPEndPoint peerEndpoint;
var cli = new CommandLineApplication
{
Name = "ditto",
FullName = "Ditto"
};
......@@ -34,7 +38,8 @@ namespace Ditto {
return 0;
});
cli.Command("get-meta", sub => {
cli.Command("get-meta", sub =>
{
sub.Description = "Fetches the metadata for the specified torrent, saving it as a .torrent file.";
sub.HelpOption("-h | --help | -?");
......@@ -45,19 +50,37 @@ namespace Ditto {
"infohash",
"The infohash of the torrent to fetch.");
sub.OnExecute(() => {
var testingArgument = sub.Argument(
"testing",
"Are you testing connectivity with a single, pre-defined peer?");
sub.OnExecute(() =>
{
GlobalLogger.LoggerFactory.AddConsole(
verboseOption.HasValue() ? LogLevel.Debug : LogLevel.Information, true);
using (var client = new Ditto.BitTorrent.Client()) {
client.Example(new IPAddress[] { IPAddress.Loopback }).Wait();
using (var client = new Ditto.BitTorrent.Client())
{
if (args.Length == 3)
{
// for testing functionality with a single controlled peer client -- pass target IP and port as args
peerIP = IPAddress.Parse(args[1]);
peerEndpoint = new IPEndPoint(peerIP, Int32.Parse(args[2]));
client.Example(new IPAddress[] { IPAddress.Loopback }, peerEndpoint).Wait();
}
else
{
client.Example(new IPAddress[] { IPAddress.Loopback }).Wait();
}
}
return 0;
return 0;
});
});
cli.Command("connect", sub => {
cli.Command("connect", sub =>
{
sub.Description = "Refreshes the DHT connection as neccessary.";
sub.HelpOption("-h | --help | -?");
......@@ -68,18 +91,22 @@ namespace Ditto {
"bootstrap",
"The address:port pairs for any bootstrap nodes to use if neccessary.");
sub.OnExecute(() => {
sub.OnExecute(() =>
{
GlobalLogger.LoggerFactory.AddConsole(
verboseOption.HasValue() ? LogLevel.Debug : LogLevel.Information, true);
using (var client = new Ditto.BitTorrent.Client()) {
using (var client = new Ditto.BitTorrent.Client())
{
client.Example(new IPAddress[] { IPAddress.Loopback }).Wait();
}
return 0;
return 0;
});
});
//string[] moreArgs = new string[args.Length - 2];
//Array.Copy(args, 2, moreArgs, 0, moreArgs.Length);
return cli.Execute(args);
}
}
......
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using System.IO;
using System.IO;
using Ditto.Common;
......
......@@ -21,7 +21,7 @@ namespace Ditto
// ourExtCode refers to the value we have associated with ut_metadata
// theirExtCode refers to the value they have associated with ut_metadata
public void RequestMetadata(NetworkStream stream, TcpClient connection, byte ourExtCode, byte theirExtCode, byte[] infohash)
public byte[] GetMetadata(NetworkStream stream, TcpClient connection, byte ourExtCode, byte theirExtCode, byte[] infohash)
{
if (!connection.Connected)
{
......@@ -88,13 +88,13 @@ namespace Ditto
combinedPieces = info.Result.Result;
} catch (BytesVerificationException ex) {
logger.LogWarning(LoggingEvents.METADATA_FAILURE, "metadata verification failed!", ex);
return;
throw new MetadataException("Metadata verification failed -- try another peer.");
}
logger.LogInformation(LoggingEvents.DATA_STORAGE_ACTION, "metadata verified! saving...");
DataHandler.SaveMetadata(combinedPieces);
logger.LogInformation(LoggingEvents.DATA_STORAGE_ACTION, "metadata saved.");
return;
return combinedPieces;
}
var request = ConstructRequestMessage(ourExtCode, currentPiece);
......@@ -133,7 +133,7 @@ namespace Ditto
}
// For messages with msgType 0 (request) and 2 (reject)
private static byte[] ConstructGenericMessage(int ourExtCode, int msgType, int piece)
private static byte[] ConstructMessage(int ourExtCode, int msgType, int piece)
{
var messageDict = new Dictionary<byte[], object>();
messageDict.Set("msg_type", msgType);
......@@ -153,12 +153,12 @@ namespace Ditto
private static byte[] ConstructRequestMessage(int ourExtCode, int piece)
{
return ConstructGenericMessage(ourExtCode, 0, piece);
return ConstructMessage(ourExtCode, 0, piece);
}
private static byte[] ConstructRejectMessage(int ourExtCode, int piece)
{
return ConstructGenericMessage(ourExtCode, 2, piece);
return ConstructMessage(ourExtCode, 2, piece);
}
// For messages with msgType 1 (data)
......@@ -167,4 +167,9 @@ namespace Ditto
return new byte[4];
}
}
public class MetadataException : Exception
{
public MetadataException(string message) : base(message) { }
}
}
using System;
using System.Collections.Generic;
using System.Text;
using System.Linq;
using System.Net;
using System.Net.Sockets;
......@@ -7,58 +8,56 @@ using System.Net.Sockets;
using Microsoft.Extensions.Logging;
using Ditto.Common;
using Ditto.BitTorrent;
namespace Ditto.PeerProtocol
{
public class PeerConnection
class PeerConnection
{
readonly private Int32 myPort = 6881;
readonly private IPAddress myIpAddress;
readonly private byte[] peerId = new byte[20].FillRandom();
private bool extensionsEnabled = false;
private bool theirExtensionsEnabled = false;
private IPEndPoint peer;
Torrent torrent;
ILogger logger { get; } = GlobalLogger.CreateLogger<PeerConnection>();
public PeerConnection(IPAddress ipAddress)
public PeerConnection(IPEndPoint peer, Torrent torrent)
{
myIpAddress = ipAddress;
"-FR0001-".ToASCII().CopyTo(peerId, 0);
this.peer = peer;
this.torrent = torrent;
}
public void InitiateHandshake(IPAddress peerIP, Int32 peerPort, byte[] infoHash)
public void InitiateHandshake(byte[] infoHash)
{
logger.LogInformation("Our peer id: " + peerId.ToHuman());
logger.LogInformation("Our Peer id: " + Client.peerId.ToHuman());
var fixedHeader = new byte[20];
fixedHeader[0] = (byte) 19;
fixedHeader[0] = (byte)19;
"BitTorrent protocol".ToASCII().CopyTo(fixedHeader, 1);
var bufferBitfield = new byte[8];
bufferBitfield[5] = (byte) 16;
extensionsEnabled = true;
bufferBitfield[5] = (byte)16;
Client.extensionsEnabled = true;
TcpClient connection = new TcpClient();
connection.ConnectAsync(peerIP, peerPort).Wait();
connection.ConnectAsync(peer.Address, peer.Port).Wait();
if (!connection.Connected)
{
throw new Exception("Failed to connect to peer.");
throw new Exception("Failed to connect to Peer.");
}
var initialHandshake = new byte[68];
fixedHeader.CopyTo(initialHandshake, 0);
bufferBitfield.CopyTo(initialHandshake, fixedHeader.Length);
infoHash.CopyTo(initialHandshake, fixedHeader.Length + bufferBitfield.Length);
peerId.CopyTo(initialHandshake, fixedHeader.Length + bufferBitfield.Length + infoHash.Length);
Client.peerId.CopyTo(initialHandshake, fixedHeader.Length + bufferBitfield.Length + infoHash.Length);
logger.LogInformation(LoggingEvents.HANDSHAKE_OUTGOING, "Sending our handshake to " + peerIP + ":" + peerPort);
logger.LogInformation(LoggingEvents.HANDSHAKE_OUTGOING, "Sending our handshake to " + peer.Address + ":" + peer.Port);
using (var stream = connection.GetStream())
{
stream.Write(initialHandshake);
logger.LogInformation(LoggingEvents.HANDSHAKE_INCOMING, "Received response from peer.");
logger.LogInformation(LoggingEvents.HANDSHAKE_INCOMING, "Received response from Peer.");
var theirFixedHeader = stream.ReadBytes(20);
if (!theirFixedHeader.SequenceEqual(fixedHeader))
{
......@@ -78,10 +77,10 @@ namespace Ditto.PeerProtocol
throw new Exception("Peer failed to return a matching infohash; aborting connection.");
}
var theirPeerId = stream.ReadBytes(20);
logger.LogInformation(LoggingEvents.HANDSHAKE_INCOMING, "The peer's ID is " + theirPeerId.ToHuman());
var theirpeerId = stream.ReadBytes(20);
logger.LogInformation(LoggingEvents.HANDSHAKE_INCOMING, "The Peer's ID is " + theirpeerId.ToHuman());
if (extensionsEnabled && theirExtensionsEnabled)
if (Client.extensionsEnabled && theirExtensionsEnabled)
{
var theirExtensionHeader = GetPeerExtensionHeader(stream);
var decodedExtensionHeader = Bencoding.DecodeDict(theirExtensionHeader);
......@@ -102,15 +101,22 @@ namespace Ditto.PeerProtocol
// Send interested message
stream.Write(1.EncodeBytes());
stream.Write(new byte[1]{2});
stream.Write(new byte[1] { 2 });
logger.LogInformation(LoggingEvents.PEER_PROTOCOL_MSG, "Sent interested message.");
if (theirExtensions.ContainsKey("ut_metadata")) {
if (theirExtensions.ContainsKey("ut_metadata"))
{
logger.LogInformation(LoggingEvents.METADATA_EXCHANGE, "They also support metadata exchange. Lets try that.");
var theirMetadataExtensionId = (byte) theirExtensions.Get("ut_metadata");
var metadata = new MetadataExchange(decodedExtensionHeader.Get("metadata_size"));
metadata.RequestMetadata(stream, connection, 2, theirMetadataExtensionId, infoHash);
var theirMetadataExtensionId = (byte)theirExtensions.Get("ut_metadata");
var metadataExchange = new MetadataExchange(decodedExtensionHeader.Get("metadata_size"));
try
{
torrent.Metadata = metadataExchange.GetMetadata(stream, connection, 2, theirMetadataExtensionId, infoHash);
} catch (MetadataException e)
{
logger.LogWarning("Unable to get metadata from current peer: ", e);
}
}
}
}
......@@ -143,12 +149,12 @@ namespace Ditto.PeerProtocol
{
var extensionDict = new Dictionary<byte[], object>();
var supportedExtensions = new Dictionary<byte[], object>();
supportedExtensions.Set("ut_metadata", 2);
extensionDict.Set("m", supportedExtensions);
// metadata_size is unnecessary if we are requesting. If we're providing metadata, we should add this.
// extensionDict.Set("metadata_size", 0);
extensionDict.Set("p", myPort);
extensionDict.Set("p", Client.myPort);
extensionDict.Set("v", "Ditto 0.1.0");
return Bencoding.Encode(extensionDict);
......
using System;
using System.Collections.Generic;
using System.Text;
using System.Net;
using Microsoft.Extensions.Logging;
using Ditto.Common;
namespace Ditto.PeerProtocol
{
public class TorrentManager
{
Torrent torrent;
Dictionary<IPEndPoint, PeerConnection> peers = new Dictionary<IPEndPoint, PeerConnection>();
ILogger logger { get; } = GlobalLogger.CreateLogger<TorrentManager>();
public TorrentManager(byte[] infohash)
{
torrent = new Torrent(infohash);
}
public TorrentManager(byte[] infohash, byte[] metadata)
{
torrent = new Torrent(infohash, metadata);
}
public void AddPeer(IPEndPoint peer)
{
var connection = new PeerConnection(peer, torrent);
peers[peer] = connection;
logger.LogInformation($"Connecting to peer {peer.Address} for {torrent.Infohash.ToHex()}");
// for now, let's assume that all connections are outgoing connections.
// we'll have a little conditional to direct incoming connections later.
connection.InitiateHandshake(torrent.Infohash);
logger.LogInformation($"Metadata is now set to: {torrent.Metadata.ToHuman()}");
}
}
}
using System;
using System.Collections.Generic;
using System.Text;
namespace Ditto.PeerProtocol
{
public class Torrent
{
public byte[] Infohash { get; set; }
public byte[] Metadata { get; set; }
public byte[][] dataPieces { get; set; } // will get the length of this from metadata
public Torrent(byte[] infohash)
{
Infohash = infohash;
}
public Torrent(byte[] infohash, byte[] metadata)
{
Infohash = infohash;
Metadata = metadata;
}
}
}
......@@ -2,7 +2,7 @@
"profiles": {
"Ditto": {
"commandName": "Project",
"commandLineArgs": "192.168.99.100"
"commandLineArgs": "192.168.1.153 8999"
}
}
}
\ No newline at end of file
using System;
using System.Net;
using Ditto.Common;
using Microsoft.Extensions.Logging;
namespace Ditto
{
// This class exists temporarily, and will definitely NOT make it into production.
// Its sole purpose is to determine if we're using the CLInterface or not
// until CLInterface is 100% functional/works with Chris's workflow.
class Runner
{
ILogger logger { get; } = GlobalLogger.CreateLogger<Runner>();
public static int Main(string[] args)
{
var useCLI = true;
GlobalLogger.LoggerFactory.AddConsole(LogLevel.Information, true);
if (useCLI)
{
CLInterface.RunCLI(args);
}
else
{
using (var client = new Ditto.BitTorrent.Client())
{
var peerIP = IPAddress.Parse(args[0]);
var peerEndpoint = new IPEndPoint(peerIP, Int32.Parse(args[1]));
client.Example(new IPAddress[] { IPAddress.Loopback }, peerEndpoint).Wait();
}
}
return 0;
}
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment