Commit 97c0f232 by Kevin C. Krinke

dscanner - Drozer based post-build dynamic vulnerability scanner command

 * New command `dscanner`, enables one to scan signed APKs with Drozer
 * Drozer is a dynamic vulnerability scanner for Android
 * Drozer runs in a emulator or on-device, this new `dscanner` command...
  * starts a docker image with Drozer and the Android Emulator pre-installed,
  * loads the signed APK into the emulator
  * activates Drozer automated tests for the APK
  * gathers the report output and places it next to the original APK
 * The Drozer docker image can be:
  * cached locally for re-use (just don't run --clean*)
  * retrieved from dockerhub.com for more efficient runtime
  * or be built from scratch (in the new "./docker" directory)
 * New "Vulnerability Scanning" documentation section (run gendocs.sh)
parent 2237b6f7
Pipeline #5049243 passed with stage
in 19 minutes 31 seconds
......@@ -67,6 +67,7 @@ via other mechanisms like Brew/dnf/pacman/emerge/Fink/MacPorts.
For Debian based distributions:
apt-get install python3-dev python3-pip python3-venv libjpeg-dev zlib1g-dev
apt-get install libffi-dev libssl-dev
Then here's how to install:
......
# This image is intended to be used with fdroidserver for the purpose
# of dynamic scanning of pre-built APKs during the fdroid build process.
# Start with ubuntu 12.04 (i386).
FROM ubuntu:14.04
MAINTAINER fdroid.dscanner <fdroid.dscanner@gmail.com>
ENV DROZER_URL https://github.com/mwrlabs/drozer/releases/download/2.3.4/drozer_2.3.4.deb
ENV DROZER_DEB drozer_2.3.4.deb
ENV AGENT_URL https://github.com/mwrlabs/drozer/releases/download/2.3.4/drozer-agent-2.3.4.apk
ENV AGENT_APK drozer-agent-2.3.4.apk
# Specially for SSH access and port redirection
ENV ROOTPASSWORD android
# Expose ADB, ADB control and VNC ports
EXPOSE 22
EXPOSE 5037
EXPOSE 5554
EXPOSE 5555
EXPOSE 5900
EXPOSE 5901
ENV DEBIAN_FRONTEND noninteractive
RUN echo "debconf shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections
RUN echo "debconf shared/accepted-oracle-license-v1-1 seen true" | debconf-set-selections
# Update packages
RUN apt-get -y update
# Drozer packages
RUN apt-get install wget python2.7 python-dev python2.7-dev python-openssl python-twisted python-protobuf bash-completion -y
# First, install add-apt-repository, sshd and bzip2
RUN apt-get -y install python-software-properties bzip2 ssh net-tools
# ubuntu 14.04 needs this too
RUN apt-get -y install software-properties-common
# Add oracle-jdk7 to repositories
RUN add-apt-repository ppa:webupd8team/java
# Make sure the package repository is up to date
RUN echo "deb http://archive.ubuntu.com/ubuntu trusty main universe" > /etc/apt/sources.list
# Update apt
RUN apt-get update
# Add drozer
RUN useradd -ms /bin/bash drozer
# Install oracle-jdk7
RUN apt-get -y install oracle-java7-installer
# Install android sdk
RUN wget http://dl.google.com/android/android-sdk_r23-linux.tgz
RUN tar -xvzf android-sdk_r23-linux.tgz
RUN mv -v android-sdk-linux /usr/local/android-sdk
# Install apache ant
RUN wget http://archive.apache.org/dist/ant/binaries/apache-ant-1.8.4-bin.tar.gz
RUN tar -xvzf apache-ant-1.8.4-bin.tar.gz
RUN mv -v apache-ant-1.8.4 /usr/local/apache-ant
# Add android tools and platform tools to PATH
ENV ANDROID_HOME /usr/local/android-sdk
ENV PATH $PATH:$ANDROID_HOME/tools
ENV PATH $PATH:$ANDROID_HOME/platform-tools
# Add ant to PATH
ENV ANT_HOME /usr/local/apache-ant
ENV PATH $PATH:$ANT_HOME/bin
# Export JAVA_HOME variable
ENV JAVA_HOME /usr/lib/jvm/java-7-oracle
# Remove compressed files.
RUN cd /; rm android-sdk_r23-linux.tgz && rm apache-ant-1.8.4-bin.tar.gz
# Some preparation before update
RUN chown -R root:root /usr/local/android-sdk/
# Install latest android tools and system images
RUN echo "y" | android update sdk --filter platform-tool --no-ui --force
RUN echo "y" | android update sdk --filter platform --no-ui --force
RUN echo "y" | android update sdk --filter build-tools-22.0.1 --no-ui -a
RUN echo "y" | android update sdk --filter sys-img-x86-android-19 --no-ui -a
#RUN echo "y" | android update sdk --filter sys-img-x86-android-21 --no-ui -a
#RUN echo "y" | android update sdk --filter sys-img-x86-android-22 --no-ui -a
RUN echo "y" | android update sdk --filter sys-img-armeabi-v7a-android-19 --no-ui -a
#RUN echo "y" | android update sdk --filter sys-img-armeabi-v7a-android-21 --no-ui -a
#RUN echo "y" | android update sdk --filter sys-img-armeabi-v7a-android-22 --no-ui -a
# Update ADB
RUN echo "y" | android update adb
# Create fake keymap file
RUN mkdir /usr/local/android-sdk/tools/keymaps
RUN touch /usr/local/android-sdk/tools/keymaps/en-us
# Run sshd
RUN apt-get install -y openssh-server
RUN mkdir /var/run/sshd
RUN echo "root:$ROOTPASSWORD" | chpasswd
RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config
RUN sed -i 's/PermitEmptyPasswords no/PermitEmptyPasswords yes/' /etc/ssh/sshd_config
# SSH login fix. Otherwise user is kicked off after login
RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
ENV NOTVISIBLE "in users profile"
RUN echo "export VISIBLE=now" >> /etc/profile
# Install socat
RUN apt-get install -y socat
# symlink android bins
RUN ln -sv /usr/local/android-sdk/tools/android /usr/local/bin/
RUN ln -sv /usr/local/android-sdk/tools/emulator /usr/local/bin/
RUN ln -sv /usr/local/android-sdk/tools/ddms /usr/local/bin/
RUN ln -sv /usr/local/android-sdk/tools/scheenshot2 /usr/local/bin/
RUN ln -sv /usr/local/android-sdk/tools/monkeyrunner /usr/local/bin/
RUN ln -sv /usr/local/android-sdk/tools/monitor /usr/local/bin/
RUN ln -sv /usr/local/android-sdk/tools/mksdcard /usr/local/bin/
RUN ln -sv /usr/local/android-sdk/tools/uiautomatorviewer /usr/local/bin/
RUN ln -sv /usr/local/android-sdk/tools/traceview /usr/local/bin/
RUN ln -sv /usr/local/android-sdk/platform-tools/adb /usr/local/bin/
RUN ln -sv /usr/local/android-sdk/platform-tools/fastboot /usr/local/bin/
RUN ln -sv /usr/local/android-sdk/platform-tools/sqlite3 /usr/local/bin/
# Setup DROZER...
# https://labs.mwrinfosecurity.com/tools/drozer/
# Run as drozer user
WORKDIR /home/drozer
# Site lists the shasums, however, I'm not sure the best way to integrate the
# checks here. No real idiomatic way for Dockerfile to do that and most of
# the examples online use chained commands but we want things to *BREAK* when
# the sha doesn't match. So far, I can't seem to reliably make Docker not
# finish the image build process.
# Download the console
RUN wget -c $DROZER_URL
# Install the console
RUN dpkg -i $DROZER_DEB
# Download agent
RUN wget -c $AGENT_URL
# Keep it version agnostic for other scripts such as install_drozer.py
RUN mv -v $AGENT_APK drozer-agent.apk
# Port forwarding required by drozer
RUN echo 'adb forward tcp:31415 tcp:31415' >> /home/drozer/.bashrc
# Alias for Drozer
RUN echo "alias drozer='drozer console connect'" >> /home/drozer/.bashrc
# add extra scripting
COPY install_agent.py /home/drozer/install_agent.py
RUN chmod 755 /home/drozer/install_agent.py
COPY enable_service.py /home/drozer/enable_service.py
RUN chmod 755 /home/drozer/enable_service.py
COPY drozer.py /home/drozer/drozer.py
RUN chmod 755 /home/drozer/drozer.py
# fix ownerships
RUN chown -R drozer.drozer /home/drozer
RUN apt-get -y --force-yes install python-pkg-resources=3.3-1ubuntu1
RUN apt-get -y install python-pip python-setuptools git
RUN pip install "git+https://github.com/dtmilano/AndroidViewClient.git#egg=androidviewclient"
RUN apt-get -y install python-pexpect
# Add entrypoint
COPY entrypoint.sh /home/drozer/entrypoint.sh
RUN chmod +x /home/drozer/entrypoint.sh
ENTRYPOINT ["/home/drozer/entrypoint.sh"]
SHELL := /bin/bash
ALIAS = "dscanner"
EXISTS := $(shell docker ps -a -q -f name=$(ALIAS))
RUNNED := $(shell docker ps -q -f name=$(ALIAS))
ifneq "$(RUNNED)" ""
IP := $(shell docker inspect $(ALIAS) | grep "IPAddress\"" | head -n1 | cut -d '"' -f 4)
endif
STALE_IMAGES := $(shell docker images | grep "<none>" | awk '{print($$3)}')
EMULATOR ?= "android-19"
ARCH ?= "armeabi-v7a"
COLON := :
.PHONY = build clean kill info
all: help
help:
@echo "usage: make {help|build|clean|kill|info}"
@echo ""
@echo " help this help screen"
@echo " build create docker image"
@echo " clean remove images and containers"
@echo " kill stop running containers"
@echo " info details of running container"
build:
@docker build -t "dscanner/fdroidserver:latest" .
clean: kill
@docker ps -a -q | xargs -n 1 -I {} docker rm -f {}
ifneq "$(STALE_IMAGES)" ""
@docker rmi -f $(STALE_IMAGES)
endif
kill:
ifneq "$(RUNNED)" ""
@docker kill $(ALIAS)
endif
info:
@docker ps -a -f name=$(ALIAS)
ifneq "$(RUNNED)" ""
$(eval ADBPORT := $(shell docker port $(ALIAS) | grep '5555/tcp' | awk '{split($$3,a,"$(COLON)");print a[2]}'))
@echo -e "Use:\n adb kill-server\n adb connect $(IP):$(ADBPORT)"
else
@echo "Run container"
endif
# dscanner docker image #
Use `make help` for up-to-date instructions.
```
usage: make {help|build|clean|kill|info}
help this help screen
build create docker image
clean remove images and containers
kill stop running containers
info details of running container
```
import pexpect
import sys
prompt = "dz>"
target = sys.argv[1]
drozer = pexpect.spawn("drozer console connect")
drozer.logfile = open("/tmp/drozer_report.log", "w")
# start
drozer.expect(prompt)
def send_command(command, target):
cmd = "run {0} -a {1}".format(command, target)
drozer.sendline(cmd)
drozer.expect(prompt)
scanners = [
"scanner.misc.native", # Find native components included in packages
# "scanner.misc.readablefiles", # Find world-readable files in the given folder
# "scanner.misc.secretcodes", # Search for secret codes that can be used from the dialer
# "scanner.misc.sflagbinaries", # Find suid/sgid binaries in the given folder (default is /system).
# "scanner.misc.writablefiles", # Find world-writable files in the given folder
"scanner.provider.finduris", # Search for content providers that can be queried.
"scanner.provider.injection", # Test content providers for SQL injection vulnerabilities.
"scanner.provider.sqltables", # Find tables accessible through SQL injection vulnerabilities.
"scanner.provider.traversal" # Test content providers for basic directory traversal
]
for scanner in scanners:
send_command(scanner, target)
#!/usr/bin/python
from subprocess import call
import sys
from com.dtmilano.android.viewclient import ViewClient
vc = ViewClient(*ViewClient.connectToDeviceOrExit())
button = vc.findViewWithText("OFF")
if button:
(x, y) = button.getXY()
button.touch()
else:
print "Button not found. Is the app currently running?"
exit()
print "Done!"
#!/bin/bash
if [[ $EMULATOR == "" ]]; then
EMULATOR="android-19"
echo "Using default emulator $EMULATOR"
fi
if [[ $ARCH == "" ]]; then
ARCH="x86"
echo "Using default arch $ARCH"
fi
echo EMULATOR = "Requested API: ${EMULATOR} (${ARCH}) emulator."
if [[ -n $1 ]]; then
echo "Last line of file specified as non-opt/last argument:"
tail -1 $1
fi
# Run sshd
/usr/sbin/sshd
adb start-server
# Detect ip and forward ADB ports outside to outside interface
ip=$(ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f2 | awk '{ print $1}')
socat tcp-listen:5037,bind=$ip,fork tcp:127.0.0.1:5037 &
socat tcp-listen:5554,bind=$ip,fork tcp:127.0.0.1:5554 &
socat tcp-listen:5555,bind=$ip,fork tcp:127.0.0.1:5555 &
# Set up and run emulator
if [[ $ARCH == *"x86"* ]]
then
EMU="x86"
else
EMU="arm"
fi
#FASTDROID_VNC_URL="https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/fastdroid-vnc/fastdroid-vnc"
#wget -c "${FASTDROID_VNC_URL}"
export PATH="${PATH}:/usr/local/android-sdk/tools/:/usr/local/android-sdk/platform-tools/"
echo "no" | android create avd -f -n test -t ${EMULATOR} --abi default/${ARCH}
echo "no" | emulator64-${EMU} -avd test -noaudio -no-window -gpu off -verbose -qemu -usbdevice tablet -vnc :0
#!/usr/bin/python
import os
from subprocess import call, check_output
from time import sleep
FNULL = open(os.devnull, 'w')
print "Ensuring device is online"
call("adb wait-for-device", shell=True)
print "Installing the drozer agent"
print "If the device just came online it is likely the package manager hasn't booted."
print "Will try multiple attempts to install."
print "This may need tweaking depending on hardware."
attempts = 0
time_to_sleep = 30
while attempts < 8:
output = check_output('adb shell "pm list packages"', shell=True)
print "Checking whether the package manager is up..."
if "Could not access the Package Manager" in output:
print "Nope. Sleeping for 30 seconds and then trying again."
sleep(time_to_sleep)
else:
break
time_to_sleep = 5
attempts = 0
while attempts < 5:
sleep(time_to_sleep)
try:
install_output = check_output("adb install /home/drozer/drozer-agent.apk", shell=True)
except Exception:
print "Failed. Trying again."
attempts += 1
else:
attempts += 1
if "Error: Could not access the Package Manager" not in install_output:
break
print "Install attempted. Checking everything worked"
pm_list_output = check_output('adb shell "pm list packages"', shell=True)
if "com.mwr.dz" not in pm_list_output:
print install_output
exit("APK didn't install properly. Exiting.")
print "Installed ok."
print "Starting the drozer agent main activity: com.mwr.dz/.activities.MainActivity"
call('adb shell "am start com.mwr.dz/.activities.MainActivity"', shell=True, stdout=FNULL)
print "Starting the service"
# start the service
call("python /home/drozer/enable_service.py", shell=True, stdout=FNULL)
print "Forward dem ports mon."
call("adb forward tcp:31415 tcp:31415", shell=True, stdout=FNULL)
......@@ -57,6 +57,7 @@ Free Documentation License".
* Update Processing::
* Build Server::
* Signing::
* Vulnerability Scanning::
* GNU Free Documentation License::
* Index::
@end menu
......@@ -1700,6 +1701,132 @@ A new key will be generated using these details, for each application that is
built. (If a specific key is required for a particular application, this system
can be overridden using the @code{keyaliases} config settings.
@node Vulnerability Scanning
@chapter Vulnerability Scanning (dscanner)
F-Droid now includes a means of running automated vulnerability scanning
using @uref{https://github.com/mwrlabs/drozer, Drozer}. This is achieved
by starting a docker container, with the Android SDK and Emulator
prepared already, installing drozer into the emulator and scripting the
knobs to scan any fully built and signed APKs.
Note: if your application is not intended to run within an Android
emulator, please do not continue with these instructions. At this time,
the @code{dscanner} feature is fully dependent upon your application
running properly in an emulated environment.
@section Quick Start
@enumerate
@item Ensure that your application is a signed release build
@item @code{fdroid dscanner --init-only} from within the repo
@item Go for a coffee, this takes a long time and requires approximately
6 GB of disk space. Once this is complete, you'll be left with a docker
container running and ready to go.
@item @code{fdroid dscanner --latest app.pkg.name} from within the repo
to run drozer on the latest build of @code{app.pkg.name}
@item If all went well, there should be an ``app.pkg.name_CODE.apk.dscanner''
file in the repo (next to the original APK file)
@item When you're all done scanning packages, you can cleanup the docker
container with: @code{fdroid dscanner --clean-only}
@end enumerate
You can also run the drozer scan as an optional part of the overall
@code{fdroid build} operation. This option will trigger a drozer scan of
all signed APKs found in the repo. See @code{fdroid build --help} for
more information.
@section Command Line Help
@example
usage: fdroid dscanner [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]
positional arguments:
app_id app-id with optional versioncode in the form
APPID[:VERCODE]
optional arguments:
-h, --help show this help message and exit
-v, --verbose Spew out even more information than normal
-q, --quiet Restrict output to warnings and errors
-l, --latest Scan only the latest version of each package
--clean-after Clean after all scans have finished.
--clean-before Clean before the scans start and rebuild the
container.
--clean-only Clean up all containers and then exit.
--init-only Prepare drozer to run a scan
--repo-path REPO_PATH
Override repo path for built APK files.
@end example
@section From Scratch
Because the docker image used to do the Android Emulator and all of that
takes a considerable amount of time to prepare, one has been uploaded to
dockerhub.com for general use. However, the astute researcher will be
weary of any black boxes and want to build their own black box. This
section elaborates how to build the docker image yourself.
From within the F-Droid Server source code directory, @code{cd
./docker/} in order to begin.
Within this directory are the custom scripting used within the docker
image creation. For conveience, there is a simple Makefile that
wraps the process of creating images into convenient pieces.
@subsection @code{make help}
@example
usage: make help|build|clean|kill|info
help this help screen
build create docker image
clean remove images and containers
kill stop running containers
info details of running container
@end example
@subsection @code{make clean}
Stops any running containers (@code{make kill}) and then forcully
removes them from docker. After that, all images associated are also
explicitly removed.
Note: this will destroy docker images!
@subsection @code{make build}
Builds the actual docker container, tagged
``dscanner/fdroidserver:latest'' from the local directory. Obviously
this is operating with the @code{Dockerfile} to build and tie everything
together nicely.
@subsection @code{make kill}
@code{docker kill} the container tagged ``dscanner''.
@subsection @code{make info}
Prints some useful information about the currently running dscanner
container (if it is even running). The output of this command is
confusing and raw but useful none-the-less. See example output below:
@example
CONTAINER ID IMAGE COMMAND
CREATED STATUS PORTS
NAMES
b90a60afe477 dscanner/fdroidserver "/home/drozer/entrypo" 20
minutes ago Up 20 minutes 0.0.0.0:32779->22/tcp,
0.0.0.0:32778->5037/tcp, 0.0.0.0:32777->5554/tcp,
0.0.0.0:32776->5555/tcp, 0.0.0.0:32775->5900/tcp,
0.0.0.0:32774->5901/tcp dscanner
Use:
adb kill-server
adb connect 172.17.0.2:32776
@end example
Typical usage is for finding the ``adb connect'' line or the ``ssh''
port (32779 from the @code{0.0.0.0:32779->22/tcp} note).
@node GNU Free Documentation License
@appendix GNU Free Documentation License
......
......@@ -38,6 +38,7 @@ commands = {
"rewritemeta": "Rewrite all the metadata files",
"lint": "Warn about possible metadata errors",
"scanner": "Scan the source code of a package",
"dscanner": "Dynamically scan APKs post build",
"stats": "Update the stats of the repo",
"server": "Interact with the repo HTTP server",
"signindex": "Sign indexes created using update --nosign",
......
......@@ -972,6 +972,8 @@ def parse_commandline():
help="Specify that we're running on the build server")
parser.add_argument("--skip-scan", dest="skipscan", action="store_true", default=False,
help="Skip scanning the source code for binaries and other problems")
parser.add_argument("--drozer-scan", dest="dscan", action="store_true", default=False,
help="Build an emulator, install the apk on it and perform a drozer scan")
parser.add_argument("--no-tarball", dest="notarball", action="store_true", default=False,
help="Don't create a source tarball, useful when testing a build")
parser.add_argument("--no-refresh", dest="refresh", action="store_false", default=True,
......@@ -1184,6 +1186,42 @@ def main():
for fa in failed_apps:
logging.info("Build for app %s failed:\n%s" % (fa, failed_apps[fa]))
# perform a drozer scan of all successful builds
if options.dscan and build_succeeded:
from .dscanner import DockerDriver
docker = DockerDriver()
try:
for app in build_succeeded:
logging.info("Need to sign the app before we can install it.")
subprocess.call("fdroid publish {0}".format(app.id), shell=True)
apk_path = None
for f in os.listdir(repo_dir):
if f.endswith('.apk') and f.startswith(app.id):
apk_path = os.path.join(repo_dir, f)
break
if not apk_path:
raise Exception("No signed APK found at path: {0}".format(apk_path))
if not os.path.isdir(repo_dir):
exit(1)
logging.info("Performing Drozer scan on {0}.".format(app))
docker.perform_drozer_scan(apk_path, app.id, repo_dir)
except Exception as e:
logging.error(str(e))
logging.error("An exception happened. Making sure to clean up")
else:
logging.info("Scan succeeded.")
logging.info("Cleaning up after ourselves.")
docker.clean()
logging.info("Finished.")
if len(build_succeeded) > 0:
logging.info(str(len(build_succeeded)) + ' builds succeeded')
......
......@@ -36,7 +36,8 @@ setup(name='fdroidserver',
'pyasn1',
'pyasn1-modules',
'PyYAML',
'requests',
'requests < 2.11',
'docker-py == 1.9.0',
],
classifiers=[
'Development Status :: 3 - Alpha',
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment