Commit dfa9de4f authored by Sophie Brun's avatar Sophie Brun

New upstream version 1.5.1

parent 85420dc5
# ChangeLog
## 1.5.1 _(March 29, 2017)_
- `config/write_paths.yml` -- Added configurable temporary directory.
- `Parser`
- `#document` -- Updated to lazy parse the document.
- `Browser`
- `Javascript`
- `DOMMonitor` -- Don't track `setInterval()`s since we're not using them.
- `TaintTracer`
- `add_trace_to_function()` -- Catch and return on error.
- Path extractors
- `scripts` -- Fixed `nil` error.
- Plugins
- `metrics` -- Fixed type error due to race condition.
## 1.5 _(January 31, 2017)_
- Executables
......
......@@ -3,7 +3,7 @@
<table>
<tr>
<th>Version</th>
<td>1.5</td>
<td>1.5.1</td>
</tr>
<tr>
<th>Homepage</th>
......@@ -611,7 +611,7 @@ You can run `rake spec` to run **all** specs or you can run them selectively usi
**Please be warned**, the core specs will require a beast of a machine due to the
necessity to test the Grid/multi-Instance features of the system.
**Note**: _The check specs will take about 90 minutes due to the timing-attack tests._
**Note**: _The check specs will take many hours to complete due to the timing-attack tests._
## Bug reports/Feature requests
......
......@@ -197,7 +197,6 @@ end
desc 'Generate docs.'
task :docs do
outdir = "../arachni-docs"
sh "rm -rf #{outdir}"
sh "mkdir -p #{outdir}"
......@@ -207,47 +206,6 @@ task :docs do
sh "rm -rf .yardoc"
end
desc 'Generate graphics.'
task :gfx do
outdir = 'gfx/compiled'
srcdir = 'gfx/source'
sh 'mkdir -p ~/.fonts'
sh 'cp gfx/font/Beneath_the_Surface.ttf ~/.fonts'
Dir.glob( "#{srcdir}/*.svg" ).each do |src|
sh "inkscape #{src} --export-png=#{outdir}/#{File.basename( src, '.svg' )}.png"
end
cp "#{outdir}/icon.png", "#{outdir}/favicon.ico"
sh 'rm -f ~/.fonts/Beneath_the_Surface.ttf'
end
#
# Simple profiler using perftools[1].
#
# To install perftools for Ruby:
# gem install perftools.rb
#
# [1] https://github.com/tmm1/perftools.rb
#
desc 'Profile Arachni.'
task :profile do
if !Gem::Specification.find_all_by_name( 'perftools.rb' ).empty?
sh "CPUPROFILE_FREQUENCY=500 CPUPROFILE=/tmp/profile.dat " +
"RUBYOPT=\"-r`gem which perftools | tail -1`\" " +
" ./bin/arachni http://demo.testfire.net && " +
"pprof.rb --gif /tmp/profile.dat > profile.gif"
else
puts 'If you want to run the profiler please install perftools.rb first:'
puts ' gem install perftools.rb'
end
end
desc 'Remove reporter and log files.'
task :clean do
files = %w(error.log *.afr *.afs *.yaml *.json *.marshal *.gem pkg/*.gem
......
......@@ -53,7 +53,7 @@ Gem::Specification.new do |s|
s.add_dependency 'concurrent-ruby-ext', '1.0.2'
# For compressing/decompressing system state archives.
s.add_dependency 'rubyzip', '1.1.6'
s.add_dependency 'rubyzip', '1.2.1'
# HTTP proxy server
s.add_dependency 'http_parser.rb', '0.6.0'
......@@ -97,7 +97,7 @@ Gem::Specification.new do |s|
# Markup parsing, for reports and Element::XML.
s.add_dependency 'nokogiri', '1.6.8.1'
# Really fast and lightweight markup parsing, for pages.
s.add_dependency 'ox', '2.4.9'
s.add_dependency 'ox', '2.4.11'
# Outputting data in table format (arachni_rpcd_monitor).
s.add_dependency 'terminal-table', '1.4.5'
......
......@@ -16,7 +16,7 @@ class Arachni::Parser::Extractors::Scripts < Arachni::Parser::Extractors::Base
return [] if !check_for?( 'script' )
document.nodes_by_name( 'script' ).map do |s|
[s['src']].flatten.compact | from_text( s.text )
[s['src']].flatten.compact | from_text( s.text.to_s )
end
end
......
......@@ -185,42 +185,42 @@ class Arachni::Plugins::Metrics < Arachni::Plugin::Base
wait_while_framework_running
@metrics = process( @metrics )
metrics = process( @metrics )
statistics = framework.statistics
@metrics['browser_cluster']['job_time_outs'] =
metrics['browser_cluster']['job_time_outs'] =
statistics[:browser_cluster][:time_out_count]
@metrics['browser_cluster']['seconds_per_job'] =
metrics['browser_cluster']['seconds_per_job'] =
statistics[:browser_cluster][:seconds_per_job]
@metrics['browser_cluster']['total_job_time'] =
metrics['browser_cluster']['total_job_time'] =
statistics[:browser_cluster][:total_job_time]
@metrics['browser_cluster']['job_count'] =
metrics['browser_cluster']['job_count'] =
statistics[:browser_cluster][:queued_job_count]
@metrics['http']['requests'] = statistics[:http][:response_count]
metrics['http']['requests'] = statistics[:http][:response_count]
@metrics['http']['request_time_outs'] = statistics[:http][:time_out_count]
@metrics['http']['responses_per_second'] = statistics[:http][:total_responses_per_second]
metrics['http']['request_time_outs'] = statistics[:http][:time_out_count]
metrics['http']['responses_per_second'] = statistics[:http][:total_responses_per_second]
if @metrics['http']['requests'] > 0
@metrics['http']['response_time_average'] =
http_response_time_total / @metrics['http']['requests']
if metrics['http']['requests'] > 0
metrics['http']['response_time_average'] =
http_response_time_total / metrics['http']['requests']
@metrics['http']['response_size_average'] =
@metrics['general']['ingress_traffic'] / @metrics['http']['requests']
metrics['http']['response_size_average'] =
metrics['general']['ingress_traffic'] / metrics['http']['requests']
@metrics['http']['request_size_average'] =
@metrics['general']['egress_traffic'] / @metrics['http']['requests']
metrics['http']['request_size_average'] =
metrics['general']['egress_traffic'] / metrics['http']['requests']
end
@metrics['scan']['duration'] = statistics[:runtime]
@metrics['scan']['authenticated'] = !!Arachni::Options.session.check_url
metrics['scan']['duration'] = statistics[:runtime]
metrics['scan']['authenticated'] = !!Arachni::Options.session.check_url
register_results @metrics
register_results metrics
end
def find_swf( page )
......@@ -254,7 +254,7 @@ class Arachni::Plugins::Metrics < Arachni::Plugin::Base
Captures metrics about multiple aspects of the scan and the web application.
},
author: 'Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>',
version: '0.1.1'
version: '0.1.2'
}
end
......
......@@ -404,7 +404,7 @@ access unauthorized pages.
TemplateScope.global_data = global_data
tmpdir = "#{Arachni.tmpdir}/#{generate_token}/"
tmpdir = "#{Arachni::Options.paths.tmpdir}/#{generate_token}/"
FileUtils.rm_rf tmpdir
FileUtils.mkdir_p tmpdir
......
......@@ -13,3 +13,7 @@ framework:
# Default directory for scan snapshots generated either by the CLI
# or by RPC Instances.
snapshots:
# Directory for temporary files -- like for excess workload that's been
# offloaded to disk etc..
# Will default to the OS temporary directory.
tmpdir:
......@@ -28,12 +28,6 @@ module Arachni
GC.start( full_mark: false )
end
def tmpdir
# On MS Windows Dir.tmpdir can return the path with a shortname,
# better avoid that as it can be insonsistent with other paths.
get_long_win32_filename( Dir.tmpdir )
end
def null_device
Gem.win_platform? ? 'NUL' : '/dev/null'
end
......
......@@ -303,13 +303,6 @@ class Javascript
dom_monitor.timeouts
end
# @return [Array<Array>]
# Arguments for JS `setInterval` calls.
def intervals
return [] if !supported?
dom_monitor.intervals
end
# @param [HTTP::Request] request
# Request to process.
# @param [HTTP::Response] response
......
......@@ -26,9 +26,6 @@ var _tokenDOMMonitor = _tokenDOMMonitor || {
// Keeps track of setTimeout() calls.
timeouts: [],
// Keeps track of setInterval() calls.
intervals: [],
// Don't include these elements in the `digest` computation.
exclude_tags_from_digest: ['P'],
......@@ -156,7 +153,6 @@ var _tokenDOMMonitor = _tokenDOMMonitor || {
if( _tokenDOMMonitor.initialized ) return;
_tokenDOMMonitor.track_setTimeout();
_tokenDOMMonitor.track_setInterval();
_tokenDOMMonitor.track_addEventListener();
_tokenDOMMonitor.initialized = true
......@@ -376,17 +372,6 @@ var _tokenDOMMonitor = _tokenDOMMonitor || {
return _tokenDOMMonitor.hashCode( digest );
},
// Override setInterval() so that we'll know to wait for it to be triggered
// during DOM analysis to provide sufficient coverage.
track_setInterval: function () {
var original_setInterval = window.setInterval;
window.setInterval = function() {
_tokenDOMMonitor.intervals.push( arguments );
original_setInterval.apply( this, arguments );
};
},
// Override setTimeout() so that we'll know to wait for it to be triggered
// during DOM analysis to provide sufficient coverage.
track_setTimeout: function () {
......
......@@ -543,9 +543,15 @@ var _tokenTaintTracer = _tokenTaintTracer || {
},
add_trace_to_function: function ( object, name, object_name ){
// Don't trace a tracer.
if( _tokenTaintTracer.get_traced_function().toString() == (object[name] || '').toString() )
// object[name].toString() can fail for certain functions so play it
// safe and bail out.
try {
// Don't trace a tracer.
if( _tokenTaintTracer.get_traced_function().toString() == (object[name] || '').toString() )
return;
} catch (e) {
return;
}
var function_needle = 'function ' + name + '(';
......@@ -556,21 +562,22 @@ var _tokenTaintTracer = _tokenTaintTracer || {
// are unknown; framework-specified ones have been vetted.
if(
object == window && object[name] &&
(
// The name should be the same as the function name...
object[name].toString().substring( 0, function_needle.length ) !== function_needle ||
// .. and the prototype needs to not have any members.
(
// The name should be the same as the function name...
object[name].toString().substring( 0, function_needle.length ) !== function_needle ||
// .. and the prototype needs to not have any members.
(
object[name].prototype &&
!_tokenTaintTracer.isEmpty( object[name].prototype )
)
object[name].prototype &&
!_tokenTaintTracer.isEmpty( object[name].prototype )
)
)
) return;
object[name] = _tokenTaintTracer.get_traced_function(
object[name], object_name || _tokenTaintTracer.object_to_name( object ), name
);
},
install_tracers_from_list: function( list ) {
......
......@@ -307,27 +307,6 @@ module Auditor
Element::LinkTemplate::DOM, Element::UIInput::DOM, Element::UIForm::DOM
]
# Default audit options.
OPTIONS = {
# Elements to audit.
#
# If no elements have been passed to audit methods, candidates will be
# determined by {#each_candidate_element}.
elements: ELEMENTS_WITH_INPUTS,
dom_elements: DOM_ELEMENTS_WITH_INPUTS,
# If set to `true` the HTTP response will be analyzed for new elements.
# Be careful when enabling it, there'll be a performance penalty.
#
# If set to `false`, no training is going to occur.
#
# If set to `nil`, when the Auditor submits a form with original or
# sample values this option will be overridden to `true`
train: nil
}
# @return [Arachni::Page]
# Page object to be audited.
attr_reader :page
......@@ -487,21 +466,15 @@ module Auditor
# Passes each element prepared for audit to the block.
#
# If no element types have been specified in `opts`, it will use the elements
# from the check's {Base.info} hash.
#
# If no elements have been specified in `opts` or {Base.info}, it will use the
# elements in {OPTIONS}.
#
# @param [Array] types
# Element types to audit (see {OPTIONS}`[:elements]`).
# It will use the elements from the check's {Base.info} hash.
# If no elements have been specified it will use {ELEMENTS_WITH_INPUTS}.
#
# @yield [element]
# Each candidate DOM element.
# @yieldparam [Arachni::Capabilities::Auditable::DOM]
def each_candidate_element( types = [], &block )
types = self.class.info[:elements] if types.empty?
types = OPTIONS[:elements] if types.empty?
# Each candidate element.
# @yieldparam [Arachni::Element]
def each_candidate_element( &block )
types = self.class.elements
types = ELEMENTS_WITH_INPUTS if types.empty?
types.each do |elem|
elem = elem.type
......@@ -538,21 +511,15 @@ module Auditor
# Passes each element prepared for audit to the block.
#
# If no element types have been specified in `opts`, it will use the elements
# from the check's {Base.info} hash.
#
# If no elements have been specified in `opts` or {Base.info}, it will use the
# elements in {OPTIONS}.
#
# @param [Array] types
# Element types to audit (see {OPTIONS}`[:elements]`).
# It will use the elements from the check's {Base.info} hash.
# If no elements have been specified it will use {DOM_ELEMENTS_WITH_INPUTS}.
#
# @yield [element]
# Each candidate element.
# @yieldparam [Arachni::Element]
def each_candidate_dom_element( types = [], &block )
types = self.class.info[:elements] if types.empty?
types = OPTIONS[:dom_elements] if types.empty?
# @yieldparam [Arachni::Element::DOM]
def each_candidate_dom_element( &block )
types = self.class.elements
types = DOM_ELEMENTS_WITH_INPUTS if types.empty?
types.each do |elem|
elem = elem.type
......@@ -589,15 +556,13 @@ module Auditor
#
# Uses {#each_candidate_element} to decide which elements to audit.
#
# @see OPTIONS
# @see Arachni::Element::Capabilities::Auditable#audit
# @see #audit_signature
def audit( payloads, opts = {}, &block )
opts = OPTIONS.merge( opts )
if !block_given?
audit_signature( payloads, opts )
else
each_candidate_element( opts[:elements] ) do |e|
each_candidate_element do |e|
e.audit( payloads, opts, &block )
audited( e.coverage_id )
end
......@@ -609,11 +574,9 @@ module Auditor
#
# Uses {#each_candidate_element} to decide which elements to audit.
#
# @see OPTIONS
# @see Arachni::Element::Capabilities::Auditable#buffered_audit
def buffered_audit( payloads, opts = {}, &block )
opts = OPTIONS.merge( opts )
each_candidate_element( opts[:elements] ) do |e|
each_candidate_element do |e|
e.buffered_audit( payloads, opts, &block )
audited( e.coverage_id )
end
......@@ -624,11 +587,9 @@ module Auditor
#
# Uses {#each_candidate_element} to decide which elements to audit.
#
# @see OPTIONS
# @see Arachni::Element::Capabilities::Analyzable::Signature
def audit_signature( payloads, opts = {} )
opts = OPTIONS.merge( opts )
each_candidate_element( opts[:elements] )do |e|
each_candidate_element do |e|
e.signature_analysis( payloads, opts )
audited( e.coverage_id )
end
......@@ -638,11 +599,9 @@ module Auditor
#
# Uses {#each_candidate_element} to decide which elements to audit.
#
# @see OPTIONS
# @see Arachni::Element::Capabilities::Analyzable::Differential
def audit_differential( opts = {}, &block )
opts = OPTIONS.merge( opts )
each_candidate_element( opts[:elements] ) do |e|
each_candidate_element do |e|
e.differential_analysis( opts, &block )
audited( e.coverage_id )
end
......@@ -652,11 +611,9 @@ module Auditor
#
# Uses {#each_candidate_element} to decide which elements to audit.
#
# @see OPTIONS
# @see Arachni::Element::Capabilities::Analyzable::Timeout
def audit_timeout( payloads, opts = {} )
opts = OPTIONS.merge( opts )
each_candidate_element( opts[:elements] ) do |e|
each_candidate_element do |e|
e.timeout_analysis( payloads, opts )
audited( e.coverage_id )
end
......
......@@ -109,7 +109,7 @@ module Signature
# {Element::Capabilities::Submittable#platforms applicable platforms}
# for the {Element::Capabilities::Submittable#action resource} to be audited.
# @param [Hash] opts
# Options as described in {Arachni::Check::Auditor::OPTIONS} and
# Options as described in {Arachni::Element::Auditable::OPTIONS} and
# {SIGNATURE_OPTIONS}.
#
# @return [Bool]
......
......@@ -79,7 +79,7 @@ module Report
"Reporter '#{name}' cannot format the audit results as a String."
end
outfile = "#{Arachni.tmpdir}/#{generate_token}"
outfile = "#{Options.paths.tmpdir}/#{generate_token}"
@reporters.run( name, external_report, outfile: outfile )
IO.binread( outfile )
......
......@@ -73,7 +73,7 @@ class Message
def url=( url )
if @normalize_url || @normalize_url.nil?
@url = URI.normalize_url( url ).to_s.freeze
@url = URI.normalize( url ).to_s.freeze
else
@url = url.to_s.freeze
end
......
......@@ -286,7 +286,7 @@ class Response < Message
redirections = response.redirections.map do |redirect|
rurl = URI.to_absolute( redirect.headers['Location'],
response.effective_url )
rurl ||= response.effective_url
rurl ||= URI.normalize( response.effective_url )
# Broken redirection, skip it...
next if !rurl
......@@ -296,7 +296,7 @@ class Response < Message
code: redirect.code,
headers: redirect.headers
))
end
end.compact
return_code = response.return_code
return_message = response.return_message
......
......@@ -7,6 +7,7 @@
=end
require 'fileutils'
require 'tmpdir'
module Arachni::OptionGroups
......@@ -75,6 +76,16 @@ class Paths < Arachni::OptionGroup
File.expand_path( File.dirname( __FILE__ ) + '/../../..' ) + '/'
end
def tmpdir
if config['framework']['tmpdir'].to_s.empty?
# On MS Windows Dir.tmpdir can return the path with a shortname,
# better avoid that as it can be insonsistent with other paths.
Arachni.get_long_win32_filename( Dir.tmpdir )
else
Arachni.get_long_win32_filename( config['framework']['tmpdir'] )
end
end
def config
self.class.config
end
......
......@@ -221,7 +221,7 @@ class Options
elsif parsed.host == 'localhost' || parsed.host.start_with?( '127.' )
fail Error::ReservedHostname,
"Loopback interfaces (like #{parsed.host}) are nor supported," <<
"Loopback interfaces (like #{parsed.host}) are not supported," <<
' please use a different IP address or hostname.'
else
......
......@@ -237,16 +237,10 @@ class Parser
# `nil` if the response data wasn't {#text? text-based} or the response
# couldn't be parsed.
def document
return @document if @document
return if !text?
if from_document?
@document
else
@document = self.class.parse(
body,
whitelist: WHITELIST
)
end
@document = self.class.parse( body, filter: true )
end
# @note It will include common request headers as well headers from the HTTP
......
......@@ -17,7 +17,7 @@ class Text < Base
include WithValue
def text
@value
@value.to_s
end
def to_html( indentation = 2, level = 0 )
......
......@@ -20,7 +20,7 @@ module WithChildren
def text
txt = children.find { |n| n.is_a? Parser::Nodes::Text }
return if !txt
return '' if !txt
txt.value
end
......
......@@ -149,7 +149,7 @@ class <<self
end
def get_temporary_directory
"#{Arachni.tmpdir}/Arachni_Snapshot_#{Utilities.generate_token}/"
"#{Options.paths.tmpdir}/Arachni_Snapshot_#{Utilities.generate_token}/"
end
def extract( archive, directory )
......
......@@ -6,8 +6,6 @@
web site for more information on licensing and terms of use.
=end
require 'tmpdir'
module Arachni
module Support::Database
......@@ -96,7 +94,7 @@ class Base
def generate_filename
# Should be unique enough...
"#{Arachni.tmpdir}/#{self.class.name}_#{Process.pid}_#{object_id}_#{@filename_counter}".gsub( '::', '_' )
"#{Options.paths.tmpdir}/#{self.class.name}_#{Process.pid}_#{object_id}_#{@filename_counter}".gsub( '::', '_' )
ensure
@filename_counter += 1
end
......
......@@ -131,26 +131,6 @@ describe Arachni::Browser::Javascript::DOMMonitor do
end
end
describe '#intervals' do
it 'keeps track of setInterval() timers' do
load '/intervals'
expect(subject.intervals).to eq([
[
"function (name, value) {\n document.cookie = name + \"=post-\" + value;\n }",
2000, 'timeout1', 2000
]
])
sleep 2
expect(@browser.cookies.size).to eq(2)
expect(@browser.cookies.map { |c| c.to_s }.sort).to eq([
'timeout1=post-2000',
'timeout=pre'
].sort)
end
end
describe '#elements_with_events' do
it 'skips non visible elements' do
load '/elements_with_events/with-hidden'
......
......@@ -285,13 +285,6 @@ describe Arachni::Browser::Javascript do
end
end
describe '#intervals' do
it 'keeps track of setInterval() timers' do
@browser.load( @dom_monitor_url + 'interval-tracker' )
expect(subject.intervals).to eq(subject.dom_monitor.intervals)
end
end
describe '#has_sinks?' do
context 'when there are execution-flow sinks' do
it 'returns true' do
......
This diff is collapsed.
......@@ -412,7 +412,7 @@ describe Arachni::Issue do
targets: {
'Generic' => 'all'
},
elements: [:link, :form_dom],
elements: [:link],
shortname: 'test'
},
trusted: true,
......
......@@ -14,7 +14,7 @@ describe Arachni::OptionGroups::Paths do
end
end
let(:paths_config_file) { "#{Arachni.tmpdir}/paths-#{Process.pid}.yml" }
let(:paths_config_file) { "#{Arachni::Options.paths.tmpdir}/paths-#{Process.pid}.yml" }
%w(root arachni components logs checks reporters plugins services
path_extractors fingerprinters lib support mixins snapshots).each do |method|
......@@ -29,6 +29,28 @@ describe Arachni::OptionGroups::Paths do
it { is_expected.to respond_to "#{method}=" }
end
describe '#tmpdir' do
context 'when no tmpdir has been specified via config' do
it 'defaults to the OS tmpdir' do
expect(subject.tmpdir).to eq Arachni.get_long_win32_filename( Dir.tmpdir )
end
end
context "when #{described_class}.config['framework']['tmpdir']" do
it 'returns its value' do
allow(described_class).to receive(:config) do
{
'framework' => {
'tmpdir' => '/my/tmpdir/'
}
}
end
expect(subject.tmpdir).to eq('/my/tmpdir/')
end
end
end