Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • tbenz9/Sia
  • alexfreska/Sia
  • josephca88/Sia
  • jizhongyue/Sia
  • wincss/Sia
  • mharkus/Sia
  • whitecandy/Sia
  • jagdish.chauhan/Sia
  • nielscastien/Sia
  • pachisi456/Sia
  • shatnerz/Sia
  • starius/Sia
  • geo-gs/Sia
  • bitrocks/Sia
  • ajespoo/Sia
  • Sanasol/Sia
  • SiaClassic-Foundation/SiaClassic
  • evanlinjin/Sia
  • xurious/Sia
  • benn7/Sia
  • pichlerx/Sia
  • thelou1s/Sia
  • eaddingtonwhite/Sia
  • roiser/Sia
  • mohsinr/Sia
  • zuoyong8/Sia
  • stjordanis/Sia
  • crypto_rocket/Sia
  • rmgianni/Sia
  • marcinja/Sia
  • Danushka96/Sia
  • bmorelax/Sia
  • JoshVorick/Sia
  • admin341/Sia
  • scott.e.graves/Sia
  • hwengmgr/Sia
  • rishi.vikram.1/Sia
  • stevefunk/Sia
  • BlackSparuce/Sia
  • DaWe35/Sia
  • asoltys/Sia
  • v-i-k-r-a-m/Sia
  • reinisp/Sia
  • ericramos1980/Sia
  • n8maninger/Sia
  • jav_/Sia
  • kunalkamble/Sia
  • awnumar/Sia
  • ps2xu/Sia
  • figuetbe/Sia
  • firyx/Sia
  • xloem/Sia
  • think-biq/Sia
  • ericflo/Sia
  • loftwah/Sia
  • EvilRedHorse/Sia
  • pjbrone/Sia
  • JonConrad/Sia
  • simonboegs/Sia
  • dassio/Sia
  • Didrole/Sia
  • netzbasis/Sia
  • Ljzn/Sia
  • vlad.lazar/Sia
  • ThePiGuy2/Sia
  • saman.20202001/Sia
  • massaccesimirco/Sia
  • arthurlewisbrown/Sia
  • hantianxu1996/Sia
  • dinovasya4/Sia
  • SKsample/Sia
  • ivandeex2/Sia
72 results
Show changes
Commits on Source (17137)
Showing
with 2011 additions and 880 deletions
# vim
*.swp
*.swo
# build artifacts
analyze
artifacts
cover
fullcover
release
# pdflatex
......@@ -13,3 +17,10 @@ doc/whitepaper.pdf
# IntelliJ IDEA
.idea
sia.iml
# VSCode debug files
.vscode
debug.test
# OSX auto-generated file
**/.DS_Store
###############################################
# Define stages of the CI
###############################################
stages:
- lint
- test
- deploy
###############################################
# Define the package variables for the tests
###############################################
variables:
legacy_pkgs: >-
./node
./node/api
./node/api/server
package_host_pkgs: >-
./modules/host
./modules/host/contractmanager
./modules/host/mdm
package_renter_pkgs: >-
./modules/renter
./modules/renter/contractor
./modules/renter/filesystem
./modules/renter/filesystem/siafile
./modules/renter/filesystem/siadir
./modules/renter/hostdb
./modules/renter/hostdb/hosttree
./modules/renter/proto
./modules/renter/skynetblocklist
./modules/renter/skynetportals
package_pkgs: >-
./build
./cmd/sia-node-scanner
./cmd/siac
./cmd/siad
./cmd/skynet-benchmark
./compatibility
./crypto
./modules
./modules/accounting
./modules/consensus
./modules/explorer
./modules/feemanager
./modules/gateway
./modules/miner
./modules/wallet
./modules/transactionpool
./profile
./persist
./sync
./skykey
./types
siatest_pkgs: >-
./siatest
./siatest/accounting
./siatest/consensus
./siatest/daemon
./siatest/feemanager
./siatest/gateway
./siatest/host
./siatest/miner
./siatest/transactionpool
./siatest/wallet
siatest_renter_pkgs: >-
./siatest/renter
siatest_renter_sub_pkgs: >-
./siatest/renter/contractor
./siatest/renter/hostdb
./siatest/renterhost
docker_hub_dev_hook: https://hub.docker.com/api/build/v1/source/57afb708-6926-411b-98fc-44c79f59097c/trigger/513a6d92-9f8b-4f04-b20a-ebd51b193b5d/call/
###############################################
# Define common stage parameters
###############################################
default: &default_params
## Run tests with most recent golang version to take advantage of any perf
## improvements.
image: nebulouslabs/sia-ci
before_script:
- date
- go version
- mkdir -p .cache/gocache
- export PATH=$PATH:$CI_PROJECT_DIR/.cache/bin/
- export GOPATH="$CI_PROJECT_DIR/.cache"
## Default common test params for regular and nightly pipelines
.default_test_common: &default_test_common_params
artifacts:
name: "SiaTesting-$CI_JOB_NAME"
paths:
- $CI_PROJECT_DIR/SiaTesting
- $CI_PROJECT_DIR/cover/cover.out
when: always
after_script:
- cp -R /tmp/SiaTesting $CI_PROJECT_DIR/SiaTesting
## Default common test params for regular and nightly pipelines
## for Windows runner.
.default_test_common_windows: &default_test_common_windows_params
artifacts:
name: "SiaTesting-$CI_JOB_NAME"
paths:
- $CI_PROJECT_DIR/SiaTesting
- $CI_PROJECT_DIR/cover/cover.out
when: always
after_script:
- XCOPY "C:\Windows\Temp\SiaTesting\*" "$CI_PROJECT_DIR\SiaTesting" /S /I /Y > $null
## Default params for Windows runner
.default_windows: &default_windows_params
before_script:
- Get-Date -format r
- go version
- $env:USERNAME
- New-Item -ItemType Directory -Force .cache\gocache
- $env:PATH += ";$CI_PROJECT_DIR\.cache\bin"
- $env:PATH
- $env:GOPATH = "$CI_PROJECT_DIR\.cache"
- $env:GOPATH
## Define the default test parameters.
.default_test: &default_test_params
## Merge the contents of aliases
<<: *default_params
<<: *default_test_common_params
stage: test
## disable default tests for scheduled pipelines (nightlies)
except:
- schedules
## Define the default test parameters
## for Windows runner.
.default_windows_test: &default_test_windows_params
## Merge the contents of aliases
<<: *default_windows_params
<<: *default_test_common_windows_params
stage: test
tags:
- nebulous-windows
## disable default tests for scheduled pipelines (nightlies)
except:
- schedules
## Define the default nightly test parameters.
.default_nightly_test: &default_nightly_params
## Merge the contents of aliases
<<: *default_test_params
<<: *default_test_common_params
## Only run during scheduled pipelines.
except:
- ""
only:
- schedules
## Define the default nightly test parameters
## for Windows runner.
.default_nightly_windows_test: &default_nightly_windows_params
## Merge the contents of aliases
<<: *default_test_windows_params
<<: *default_test_common_windows_params
## Only run during scheduled pipelines.
except:
- ""
only:
- schedules
###############################################
# Define specific stage parameters
###############################################
#################
# Lint Stage
#################
lint:
stage: lint
cache:
key: lint-cache
paths:
- .cache
script:
- export GOCACHE=$CI_PROJECT_DIR/.cache/gocache
- make vet
- make lint
- make markdown-spellcheck
- make test
#################
# Build Stage
#################
build:
stage: lint
## go 1.13 is the minimum accepted version for building Sia.
image: nebulouslabs/sia-ci
artifacts:
name: "Binaries"
paths:
- $CI_PROJECT_DIR/artifacts
script:
- ./check-builds.sh
#################
# Deploy Stage
#################
deploy:
stage: deploy
script:
- curl -X POST -H "Content-Type:application/json" --data '{"name":"'"$CI_COMMIT_REF_NAME"'"}' $docker_hub_dev_hook
only:
- master
############################
# Linux Tests
############################
#################
# Legacy Test
#################
legacy-tests:
<<: *default_test_params
cache:
key: legacy-cache
paths:
- .cache
script:
- make test-long run=. pkgs="$legacy_pkgs"
#################
# Package Tests
#################
package-host-tests:
<<: *default_test_params
cache:
key: package-cache
paths:
- .cache
script:
- make test-long run=. pkgs="$package_host_pkgs"
package-renter-tests:
<<: *default_test_params
cache:
key: package-cache
paths:
- .cache
script:
- make test-long run=. pkgs="$package_renter_pkgs"
package-tests:
<<: *default_test_params
cache:
key: package-cache
paths:
- .cache
script:
- make test-long run=. pkgs="$package_pkgs"
#################
# Siatest Tests
#################
siatest-tests:
<<: *default_test_params
cache:
key: siatest-cache
paths:
- .cache
script:
- make test-long run=. pkgs="$siatest_pkgs"
siatest-renter-tests:
<<: *default_test_params
cache:
key: siatest-renter-tests-cache
paths:
- .cache
script:
- apt-get update
- apt-get install -y fuse
- make test-long run=. pkgs="$siatest_renter_pkgs"
siatest-renter-sub-tests:
<<: *default_test_params
cache:
key: siatest-renter-sub-tests-cache
paths:
- .cache
script:
- apt-get update
- apt-get install -y fuse
- make test-long run=. pkgs="$siatest_renter_sub_pkgs"
############################
# Nightly Tests
############################
#################
# Legacy Test
#################
legacy-tests-nightly:
<<: *default_nightly_params
script:
- make test-vlong run=. pkgs="$legacy_pkgs"
# Windows execution
legacy-windows-tests-nightly:
<<: *default_nightly_windows_params
script:
- make test-vlong-windows run=. pkgs="$legacy_pkgs"
#################
# Package Test
#################
package-host-tests-nightly:
<<: *default_nightly_params
script:
- make test-vlong run=. pkgs="$package_host_pkgs"
# Windows execution
package-host-windows-tests-nightly:
<<: *default_nightly_windows_params
script:
- make test-vlong-windows run=. pkgs="$package_host_pkgs"
package-renter-tests-nightly:
<<: *default_nightly_params
script:
- make test-vlong run=. pkgs="$package_renter_pkgs"
# Windows execution
package-renter-windows-tests-nightly:
<<: *default_nightly_windows_params
script:
- make test-vlong-windows run=. pkgs="$package_renter_pkgs"
package-tests-nightly:
<<: *default_nightly_params
script:
- make test-vlong run=. pkgs="$package_pkgs"
# Windows execution
package-windows-tests-nightly:
<<: *default_nightly_windows_params
script:
- make test-vlong-windows run=. pkgs="$package_pkgs"
#################
# Siatest Test
#################
siatest-tests-nightly:
<<: *default_nightly_params
script:
- make test-vlong run=. pkgs="$siatest_pkgs"
# Windows execution
siatest-windows-tests-nightly:
<<: *default_nightly_windows_params
script:
- make test-vlong-windows run=. pkgs="$siatest_pkgs"
siatest-renter-tests-nightly:
<<: *default_nightly_params
script:
- apt-get update
- apt-get install -y fuse
- make test-vlong run=. pkgs="$siatest_renter_pkgs"
# Windows execution
siatest-renter-windows-tests-nightly:
<<: *default_nightly_windows_params
script:
- make test-vlong-windows run=. pkgs="$siatest_renter_pkgs"
siatest-renter-sub-tests-nightly:
<<: *default_nightly_params
script:
- apt-get update
- apt-get install -y fuse
- make test-vlong run=. pkgs="$siatest_renter_sub_pkgs"
# Windows execution
siatest-renter-sub-windows-tests-nightly:
<<: *default_nightly_windows_params
script:
- make test-vlong-windows run=. pkgs="$siatest_renter_sub_pkgs"
############################
# Antfarm Tests
############################
# TODO:
# Enable antfarm test triger when the following permission issue is fixed:
# https://gitlab.com/gitlab-org/gitlab/-/issues/299433
# Updates to master and nightly schedule pipelines will trigger Sia Antfarm
# version tests
# antfarm-version-tests:
# stage: test
# trigger:
# project: NebulousLabs/Sia-Ant-Farm
# # strategy: depend
# variables:
# PARENT_CI_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE
# only:
# - master
Interested in contributing to Sia?
==================================
Please review the contributing guidelines in the following pages:
- [Guide to Contributing to Sia](https://gitlab.com/NebulousLabs/Sia/blob/master/doc/Guide%20to%20Contributing%20to%20Sia.md)
- [Developers](https://gitlab.com/NebulousLabs/Sia/blob/master/doc/Developers.md)
- [All Documentation](https://gitlab.com/NebulousLabs/Sia/tree/master/doc)
# BUG REPORT
## Stack Trace or error message
## Expected Behavior
## Observed Behavior
## How to reproduce it (as minimally and precisely as possible)
## Environment
* Sia version:
* OS:
# FEATURE REQUEST
## Description of Request
## Reason or Need for Feature
## Design / Proposal
# PROGRAMMING TASK
## Description of Task
## Reason or Need for Change
## Design / Proposal
# Test NDF
## Stack Trace or error message
## Failed Pipeline
# MINOR MERGE REQUEST
## Overview
## Issues Closed
<!--
Use the `Closes` keyword to automatically close the issue on merge.
Example: Closes #XXXX
-->
# Based off of the example file at https://github.com/golangci/golangci-lint
# options for analysis running
run:
# default concurrency is a available CPU number
concurrency: 4
# timeout for analysis, e.g. 30s, 5m, default is 1m
timeout: 600s
# exit code when at least one issue was found, default is 1
issues-exit-code: 1
# include test files or not, default is true
tests: true
# list of build tags, all linters use it. Default is empty list.
build-tags: []
# which dirs to skip: issues from them won't be reported;
# can use regexp here: generated.*, regexp is applied on full path;
# default value is empty list, but default dirs are skipped independently
# from this option's value (see skip-dirs-use-default).
skip-dirs:
- cover
# default is true. Enables skipping of directories:
# vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
skip-dirs-use-default: true
# which files to skip: they will be analyzed, but issues from them
# won't be reported. Default value is empty list, but there is
# no need to include all autogenerated files, we confidently recognize
# autogenerated files. If it's not please let us know.
skip-files: []
# output configuration options
output:
# colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
format: colored-line-number
# print lines of code with issue, default is true
print-issued-lines: true
# print linter name in the end of issue text, default is true
print-linter-name: true
# all available settings of specific linters
linters-settings:
## Enabled linters:
govet:
# report about shadowed variables
check-shadowing: false
disable-all: false
golint:
min-confidence: 1.0
gocritic:
# Which checks should be enabled; can't be combined with 'disabled-checks';
# See https://go-critic.github.io/overview#checks-overview
# To check which checks are enabled run `GL_DEBUG=gocritic golangci-lint run`
# By default list of stable checks is used.
enabled-checks:
- argOrder # Diagnostic options
- badCond
- caseOrder
- dupArg
- dupBranchBody
- dupCase
- dupSubExpr
- nilValReturn
- offBy1
- weakCond
- boolExprSimplify # Style options here and below.
- builtinShadow
- emptyFallthrough
- hexLiteral
- ptrToRefParam
- underef
- equalFold
linters:
disable-all: true
fast: false
enable:
- gocritic
- gofmt
- golint
- gosec
- govet
- misspell
- typecheck
- whitespace
issues:
# Maximum issues count per one linter. Set to 0 to disable. Default is 50.
max-issues-per-linter: 0
# Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
max-same-issues: 0
# List of regexps of issue texts to exclude, empty list by default.
# But independently from this option we use default exclude patterns,
# it can be disabled by `exclude-use-default: false`. To list all
# excluded by default patterns execute `golangci-lint run --help`
exclude: []
# Independently from option `exclude` we use default exclude patterns,
# it can be disabled by this option. To list all
# excluded by default patterns execute `golangci-lint run --help`.
# Default value for this option is true.
exclude-use-default: false
# Additional exclusion rules.
exclude-rules:
# Ignore gosec hardcoded credentials false positive.
- path: cmd/siac/
text: "G101: Potential hardcoded credentials"
linters:
- gosec
- path: cmd/siac/
text: "G107: Potential HTTP request made with variable url"
linters:
- gosec
- path: build/
text: "G110: Potential DoS vulnerability via decompression bomb"
linters:
- gosec
# Some default exclude patterns that we want to keep after setting
# `exclude-use-default` to `false`.
- text: "G103: Use of unsafe calls should be audited"
linters:
- gosec
- text: "G104: Errors unhandled"
linters:
- gosec
- text: "G301: Expect directory permissions to be 0750 or less"
linters:
- gosec
# NOTE: our use of constants avoids these errors while allowing for file
# permissions higher than 0600
- text: "G302: Expect file permissions to be 0600 or less"
linters:
- gosec
- text: "G304: Potential file inclusion via variable"
linters:
- gosec
language: go
go:
- 1.5.1
install:
- make dependencies
- test -z "$(go fmt ./...)"
- make
script: make test-long && make cover && make bench
sudo: false
This diff is collapsed.
# Contributing to Sia
#### Table of Contents
* [Get started with Go](#get-started-with-go)
* [Install Go](#install-go)
* [Learn Go](#learn-go)
* [Build Sia](#build-sia)
* [Contribute to the codebase](#contribute-to-the-codebase)
* [Set up git](#set-up-git)
* [Fork the Sia repository](#fork-the-sia-repository)
* [Write some code](#write-some-code)
* [Submit your code for review](#submit-your-code-for-review)
* [More git resources](#more-git-resources)
* [Where to start](#where-to-start)
* [Contact us](#contact-us)
## Get started with Go
### Install Go
To install Go on your computer, follow the
[official installation guide][install-go].
You should install the latest [official Go binary][binary] for your system (if
not available, [install from source][source]). If you plan to cross compile
Sia, see [Cross Compilation with Go 1.5][cross] by Dave Cheney.
### Learn Go
* To get familiar with the language, start with the official [Tour of Go][tour].
* Move onto [How to Write Go Code][how] to learn how to organize Go packages
and use the go tool.
* Finish with the [Effective Go][effective] guide.
## Build Sia
To build Sia on your machine, enter the following on the command line:
```bash
# Download Sia and its dependencies
# Binaries will be installed in $GOPATH/bin
$ go get -u gitlab.com/NebulousLabs/Sia/...
# Switch to directory containing Sia source code
$ cd $GOPATH/src/gitlab.com/NebulousLabs/Sia
# You have three Sia builds to choose from.
# To build the standard release binary:
$ make release
# Or to build the release binary with race detection and an array debugging
# asserts:
$ make release-race
# Or to build the developer binary (with a different genesis block, faster
# block times, and other changes):
$ make dev
# Or build the developer binary with race detection:
$ make dev-race
# Build the debugger binary:
$ make debug
# Or build debugger binary with race detection:
$ make debug-race
```
## Contribute to the codebase
### Set up git
Install git on your machine according to [these instructions][install-git] in
the Pro Git book.
You will first need to set up global settings using the command line.
```bash
$ git config --global user.name "Your Name"
$ git config --global user.email you@somedomain.com
# Tell git to remember your login information for a certain amount of time.
# Default time is 15 minutes:
$ git config --global credential.helper cache
# Or you can choose a different amount of time:
$ git config --global credential.helper "cache --timeout=[seconds]"
```
### Fork the Sia repository
While logged into your GitLab account, navigate to the [Sia repository][sia]
and click the 'Fork' button in the upper righthand corner. Your account now
has a 'forked' copy of the original repo at
`https://gitlab.com/<your GitLab username>/Sia`.
When you installed Sia using `go get`, the go tool put the Sia source code in
$GOPATH/src/gitlab.com/NebulousLabs/Sia. Change to that directory and set up
your fork as a git [remote][remote]:
```bash
$ cd $GOPATH/src/gitlab.com/NebulousLabs/Sia
# Add your fork as a remote. Name it whatever is convenient,
# e.g your GitLab username
$ git remote add <remote name> https://gitlab.com/<username>/Sia.git
# Or if you use an SSH key, create the remote with the following
$ git remote add <remote name> git@gitlab.com:<username>/Sia.git
```
### Write some code
Right now your git local repository only has one branch (called 'master' by
default). If you want to make changes, add a new branch and make your changes
there. You should maintain master as an up-to-date copy of the NebulousLabs/Sia
repository's master branch.
To create and checkout a new branch:
```bash
# If you're not already in the right directory:
$ cd $GOPATH/src/gitlab.com/NebulousLabs/Sia
# Make sure you're on branch master
$ git checkout master
# Create and checkout a new branch
$ git checkout -b <branch>
```
Now write some code while the new branch is checked out.
Only implement one logical change per branch. If you're working on several
things at once, make multiple branches. To switch between branches you're
working on, you have to stash the changes in the branch you're switching from by
running `git stash`, which tucks away all changes since the last commit.
```bash
# Stash changes to current branch.
$ git stash
# Checkout other branch.
$ git checkout <branch>
...
# Make changes
...
# Return to first branch:
$ git checkout <branch 1>
# View a list of stashes and their corresponding hashes.
$ git stash list
# Reapply changes from the stash you want to recover and remove that stash from.
# the list
$ git stash pop <hash>
```
To learn more about branching, see [Using the Fork-and-Branch Git
Workflow][branch] and [Pro Git - Branches in a Nutshell][nutshell]. For more on
stashing, see [Pro Git - Stashing and Cleaning][stashing].
Be sure to follow the conventions detailed in
[docs/Developers.md][developers.md] and [Merge
Requests.md](./docs/Merge%20Requests.md). We will reject merge requests that do
not satisfy these best practices.
Once you've finished making changes, stage and commit your changes then update
your fork on GitLab:
```bash
# Make sure the code is up to date with the original repo:
$ git checkout master
$ git pull origin master
# Checkout branch with changes.
$ git checkout <branch>
$ git rebase master
# Before every merge request, you should run `make test-long`
# to test your code and fix formatting and style problems.
$ make test-long
# If all goes well, proceed to staging your changed files:
$ git add <changed files>
# Use `git status` to see what files have been staged.
$ git status
# Commit your changes. If you just run `commit`, a text editor will pop up for
# you to enter a description of your changes.
$ git commit -m "Add new tests for CommitSync method"
# Push the changes to your fork on GitLab, which you should have set up as a
# remote already.
$ git push <fork remote> <branch>
```
### Submit your code for review
Once you've tested your new code and pushed changes to your fork, navigate to
your fork at `https://gitlab.com/<username>/Sia` in your browser.
Switch to the branch you've made changes on by selecting it from the list on the
upper left. Then click 'New merge request' on the upper right.
Once you have made the merge request, we will review your code. We will reject
code that is unsafe, difficult to read, or otherwise violates the conventions
outlined in our [Developers][developers.md] document.
Here's a sample code review comment:
![Screenshot](doc/assets/codereview.png)
If you want to tweak code for which you've already submitted a merge request,
push the updated code to your fork with `git push -f <fork remote> <branch>` and
summarize the changes you've made in a comment on the merge request page on
GitLab.
Once we have accepted your changes and merged them into the original repo, you
have some cleanup to do:
```bash
# Update local master branch to reflect changes in origin (the original
# repo).
$ git pull origin master
# Delete the branch you made the merge request from.
$ git branch -d <branch>
# Delete the remote branch on your fork.
$ git push <fork remote> :<branch>
# Update your fork.
$ git push <fork remote> master
```
### More Git resources
* [How to into git (and GitHub)][luke] by Luke Champine
* [Official resources for learning Git][git]
## Where to start
If you'd like to contribute to Sia but don't have any specific ideas, writing
tests is a good way to get your feet wet. See [Running and Writing Tests for Sia](doc/Running%20and%20Writing%20Tests%20for%20Sia.md) to get started.
To learn more about how various parts of the code base work, head over to our [Resources](resources.md) page in our [doc](docs) folder.
## Contact us
Feel free to ask for help on the #core-dev channel on [discord][discord].
[binary]: https://golang.org/dl/
[branch]: http://blog.scottlowe.org/2015/01/27/using-fork-branch-git-workflow/
[cheney]: http://dave.cheney.net/2013/06/09/writing-table-driven-tests-in-go
[cross]: http://dave.cheney.net/2015/08/22/cross-compilation-with-go-1-5
[developers.md]: https://gitlab.com/NebulousLabs/Sia/blob/master/doc/Developers.md
[discord]: https://discord.gg/sia
[docs]: https://gitlab.com/NebulousLabs/Sia/tree/master/doc
[effective]: https://golang.org/doc/effective_go.html
[git]: https://git-scm.com/doc
[gofmt]: https://golang.org/cmd/gofmt/
[how]: https://golang.org/doc/code.html
[install-git]: https://git-scm.com/book/en/v2/Getting-Started-Installing-Git
[install-go]: https://golang.org/doc/install
[luke]: https://gist.github.com/lukechampine/6418449
[nutshell]: https://git-scm.com/book/en/v2/Git-Branching-Branches-in-a-Nutshell
[resources.md]: https://gitlab.com/NebulousLabs/Sia/blob/master/doc/Resources.md
[remote]: https://git-scm.com/book/en/v2/Git-Basics-Working-with-Remotes
[sia]: https://gitlab.com/NebulousLabs/Sia
[signup]: https://github.com/join?source=header-home
[source]: https://golang.org/doc/install/source
[stashing]: https://git-scm.com/book/en/v2/Git-Tools-Stashing-and-Cleaning
[test-doc]: https://gitlab.com/NebulousLabs/Sia/blob/master/doc/Testing.md
[tour]: https://tour.golang.org/welcome/1
The MIT License (MIT)
Copyright (c) 2015 Nebulous Inc.
Copyright (c) 2016 Nebulous Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
......
# all will build and install developer binaries, which have debugging enabled
# and much faster mining and block constants.
all: install
# These variables get inserted into ./build/commit.go
BUILD_TIME=$(shell date)
GIT_REVISION=$(shell git rev-parse --short HEAD)
GIT_DIRTY=$(shell git diff-index --quiet HEAD -- || echo "✗-")
# dependencies installs all of the dependencies that are required for building
# Sia.
ldflags= -X gitlab.com/NebulousLabs/Sia/build.GitRevision=${GIT_DIRTY}${GIT_REVISION} \
-X "gitlab.com/NebulousLabs/Sia/build.BuildTime=${BUILD_TIME}"
racevars= history_size=3 halt_on_error=1 atexit_sleep_ms=2000
# all will build and install release binaries
all: release
# count says how many times to run the tests.
count = 1
# cpkg determines which package is the target when running 'make fullcover'.
# 'make fullcover' can only provide full coverage statistics on a single package
# at a time, unfortunately.
cpkg = ./modules/renter
# pkgs changes which packages the makefile calls operate on. run changes which
# tests are run during testing.
pkgs = \
./benchmark \
./build \
./cmd/sia-node-scanner \
./cmd/siac \
./cmd/siad \
./cmd/skynet-benchmark \
./compatibility \
./crypto \
./modules \
./modules/accounting \
./modules/consensus \
./modules/explorer \
./modules/feemanager \
./modules/gateway \
./modules/host \
./modules/host/contractmanager \
./modules/host/mdm \
./modules/host/registry \
./modules/miner \
./modules/renter \
./modules/renter/contractor \
./modules/renter/filesystem \
./modules/renter/filesystem/siadir \
./modules/renter/filesystem/siafile \
./modules/renter/hostdb \
./modules/renter/hostdb/hosttree \
./modules/renter/proto \
./modules/renter/skynetblocklist \
./modules/renter/skynetportals \
./modules/transactionpool \
./modules/wallet \
./node \
./node/api \
./node/api/server \
./node/api/client \
./persist \
./profile \
./siatest \
./siatest/accounting \
./siatest/consensus \
./siatest/daemon \
./siatest/dependencies \
./siatest/feemanager \
./siatest/gateway \
./siatest/host \
./siatest/miner \
./siatest/renter \
./siatest/renter/contractor \
./siatest/renter/hostdb \
./siatest/renterhost \
./siatest/transactionpool \
./siatest/wallet \
./skykey \
./sync \
./types \
./types/typesutil \
# release-pkgs determine which packages are built for release and distribution
# when running a 'make release' command.
release-pkgs = ./cmd/siac ./cmd/siad
# lockcheckpkgs are the packages that are checked for locking violations.
lockcheckpkgs = \
./benchmark \
./build \
./cmd/sia-node-scanner \
./cmd/siac \
./cmd/siad \
./cmd/skynet-benchmark \
./node \
./node/api \
./node/api/client \
./node/api/server \
./modules/accounting \
./modules/host/mdm \
./modules/host/registry \
./modules/renter/hostdb \
./modules/renter/proto \
./modules/renter/skynetblocklist \
./siatest/accounting \
./skykey \
./types \
./types/typesutil \
# run determines which tests run when running any variation of 'make test'.
run = .
# util-pkgs determine the set of packages that are built when running
# 'make utils'
util-pkgs = ./cmd/sia-node-scanner ./cmd/skynet-benchmark
# dependencies list all packages needed to run make commands used to build, test
# and lint siac/siad locally and in CI systems.
dependencies:
# Consensus Dependencies
go get -u github.com/NebulousLabs/demotemutex
go get -u github.com/NebulousLabs/ed25519
go get -u github.com/NebulousLabs/merkletree
go get -u github.com/NebulousLabs/bolt
go get -u github.com/dchest/blake2b
go get -u golang.org/x/crypto/twofish
# Module + Daemon Dependencies
go get -u github.com/NebulousLabs/entropy-mnemonics
go get -u github.com/NebulousLabs/go-upnp
go get -u github.com/inconshreveable/go-update
go get -u github.com/inconshreveable/muxado
go get -u github.com/kardianos/osext
go get -u github.com/klauspost/reedsolomon
go get -u github.com/stretchr/graceful
# Frontend Dependencies
go get -u github.com/bgentry/speakeasy
go get -u github.com/spf13/cobra
# Developer Dependencies
go install -race std
go get -u github.com/laher/goxc
go get -u golang.org/x/tools/cmd/cover
go get -d ./...
./install-dependencies.sh
# fmt calls go fmt on all packages.
fmt:
go fmt ./...
gofmt -s -l -w $(pkgs)
# vet calls go vet on all packages.
# NOTE: go vet requires packages to be built in order to obtain type info.
vet:
go vet $(pkgs)
# REBUILD touches all of the build-dependent source files, forcing them to be
# rebuilt. This is necessary because the go tool is not smart enough to trigger
# a rebuild when build tags have been changed.
REBUILD:
@touch build/*.go
# markdown-spellcheck runs codespell on all markdown files that are not
# vendored.
markdown-spellcheck:
git ls-files "*.md" :\!:"vendor/**" | xargs codespell --check-filenames
# install builds and installs developer binaries.
install: REBUILD
go install -race -tags='dev debug profile' ./...
# lint runs golangci-lint (which includes golint, a spellcheck of the codebase,
# and other linters), the custom analyzers, and also a markdown spellchecker.
lint: markdown-spellcheck
golangci-lint run -c .golangci.yml ./...
analyze -lockcheck=false -- $(pkgs)
analyze -lockcheck -- $(lockcheckpkgs)
# spellcheck checks for misspelled words in comments or strings.
spellcheck: markdown-spellcheck
golangci-lint run -c .golangci.yml -E misspell
# staticcheck runs the staticcheck tool
# NOTE: this is not yet enabled in the CI system.
staticcheck:
staticcheck $(pkgs)
# debug builds and installs debug binaries. This will also install the utils.
debug:
go install -tags='debug profile netgo' -ldflags='$(ldflags)' $(pkgs)
debug-race:
GORACE='$(racevars)' go install -race -tags='debug profile netgo' -ldflags='$(ldflags)' $(pkgs)
# dev builds and installs developer binaries. This will also install the utils.
dev:
go install -tags='dev debug profile netgo' -ldflags='$(ldflags)' $(pkgs)
dev-race:
GORACE='$(racevars)' go install -race -tags='dev debug profile netgo' -ldflags='$(ldflags)' $(pkgs)
# release builds and installs release binaries.
release: REBUILD
go install -a -race -tags='debug profile' ./...
release-std: REBUILD
go install -a ./...
# xc builds and packages release binaries for all systems by using goxc.
# Cross Compile - makes binaries for windows, linux, and mac, 32 and 64 bit.
xc: dependencies test test-long REBUILD
goxc -arch="386 amd64 arm" -bc="darwin linux windows" -d=release \
-pv=v0.4.7 -br=beta -include=LICENSE,README.md,doc/API.md \
-main-dirs-exclude=siae -tasks-=deb,deb-dev,deb-source,go-test \
-n=Sia
release:
go install -tags='netgo' -ldflags='-s -w $(ldflags)' $(release-pkgs)
release-race:
GORACE='$(racevars)' go install -race -tags='netgo' -ldflags='-s -w $(ldflags)' $(release-pkgs)
release-util:
go install -tags='netgo' -ldflags='-s -w $(ldflags)' $(release-pkgs) $(util-pkgs)
# clean removes all directories that get automatically created during
# development.
clean:
rm -rf release doc/whitepaper.aux doc/whitepaper.log doc/whitepaper.pdf
# 3 commands and a variable are available for testing Sia packages. 'pkgs'
# indicates which packages should be tested, and defaults to all the packages
# with test files. Using './...' as default breaks compatibility with the cover
# command. 'test' runs short tests that should last no more than a few seconds,
# 'test-long' runs more thorough tests which should not last more than a few
# minutes.
run = Test
pkgs = ./api ./build ./compatibility ./crypto ./encoding ./modules ./modules/consensus \
./modules/explorer ./modules/gateway ./modules/host ./modules/hostdb \
./modules/miner ./modules/renter ./modules/transactionpool \
./modules/wallet ./persist ./siac ./siae ./sync ./types
test: REBUILD
go test -short -tags='debug testing' -timeout=3s $(pkgs) -run=$(run)
test-v: REBUILD
go test -race -v -short -tags='debug testing' -timeout=15s $(pkgs) -run=$(run)
test-long: clean fmt REBUILD
go test -v -race -tags='testing debug' -timeout=300s $(pkgs) -run=$(run)
bench: clean fmt REBUILD
go test -tags='testing' -timeout=300s -run=XXX -bench=. $(pkgs)
cover: clean REBUILD
@mkdir -p cover/modules
@for package in $(pkgs); do \
go test -tags='testing debug' -timeout=360s -covermode=atomic -coverprofile=cover/$$package.out ./$$package \
&& go tool cover -html=cover/$$package.out -o=cover/$$package.html \
&& rm cover/$$package.out ; \
done
cover-integration: clean REBUILD
@mkdir -p cover/modules
@for package in $(pkgs); do \
go test -run=TestIntegration -tags='testing debug' -timeout=300s -covermode=atomic -coverprofile=cover/$$package.out ./$$package \
&& go tool cover -html=cover/$$package.out -o=cover/$$package.html \
&& rm cover/$$package.out ; \
ifneq ("$(OS)","Windows_NT")
# Linux
rm -rf cover doc/whitepaper.aux doc/whitepaper.log doc/whitepaper.pdf fullcover release
else
# Windows
- DEL /F /Q cover doc\whitepaper.aux doc\whitepaper.log doc\whitepaper.pdf fullcover release
endif
test:
go test -short -tags='debug testing netgo' -timeout=5s $(pkgs) -run=$(run) -count=$(count)
test-v:
GORACE='$(racevars)' go test -race -v -short -tags='debug testing netgo' -timeout=15s $(pkgs) -run=$(run) -count=$(count)
test-long: clean fmt vet lint
@mkdir -p cover
GORACE='$(racevars)' go test -race --coverprofile='./cover/cover.out' -v -failfast -tags='testing debug netgo' -timeout=3600s $(pkgs) -run=$(run) -count=$(count)
# Use on Linux (and MacOS)
test-vlong: clean fmt vet lint
@mkdir -p cover
GORACE='$(racevars)' go test --coverprofile='./cover/cover.out' -v -race -tags='testing debug vlong netgo' -timeout=20000s $(pkgs) -run=$(run) -count=$(count)
# Use on Windows without fmt, vet, lint
test-vlong-windows: clean
MD cover
SET GORACE='$(racevars)'
go test --coverprofile='./cover/cover.out' -v -race -tags='testing debug vlong netgo' -timeout=20000s $(pkgs) -run=$(run) -count=$(count)
test-cpu:
go test -v -tags='testing debug netgo' -timeout=500s -cpuprofile cpu.prof $(pkgs) -run=$(run) -count=$(count)
test-mem:
go test -v -tags='testing debug netgo' -timeout=500s -memprofile mem.prof $(pkgs) -run=$(run) -count=$(count)
bench: clean fmt
go test -tags='debug testing netgo' -timeout=500s -run=XXX -bench=$(run) $(pkgs) -count=$(count)
cover: clean
@mkdir -p cover
@for package in $(pkgs); do \
mkdir -p `dirname cover/$$package` \
&& go test -tags='testing debug netgo' -timeout=500s -covermode=atomic -coverprofile=cover/$$package.out ./$$package -run=$(run) || true \
&& go tool cover -html=cover/$$package.out -o=cover/$$package.html ; \
done
cover-unit: clean REBUILD
@mkdir -p cover/modules
@for package in $(pkgs); do \
go test -run=TestUnit -tags='testing debug' -timeout=300s -covermode=atomic -coverprofile=cover/$$package.out ./$$package \
&& go tool cover -html=cover/$$package.out -o=cover/$$package.html \
&& rm cover/$$package.out ; \
# fullcover is a command that will give the full coverage statistics for a
# package. Unlike the 'cover' command, full cover will include the testing
# coverage that is provided by all tests in all packages on the target package.
# Only one package can be targeted at a time. Use 'cpkg' as the variable for the
# target package, 'pkgs' as the variable for the packages running the tests.
#
# NOTE: this command has to run the full test suite to get output for a single
# package. Ideally we could get the output for all packages when running the
# full test suite.
#
# NOTE: This command will not skip testing packages that do not run code in the
# target package at all. For example, none of the tests in the 'sync' package
# will provide any coverage to the renter package. The command will not detect
# this and will run all of the sync package tests anyway.
fullcover: clean
@mkdir -p fullcover
@mkdir -p fullcover/tests
@echo "mode: atomic" >> fullcover/fullcover.out
@for package in $(pkgs); do \
mkdir -p `dirname fullcover/tests/$$package` \
&& go test -tags='testing debug netgo' -timeout=500s -covermode=atomic -coverprofile=fullcover/tests/$$package.out -coverpkg $(cpkg) ./$$package -run=$(run) || true \
&& go tool cover -html=fullcover/tests/$$package.out -o=fullcover/tests/$$package.html \
&& tail -n +2 fullcover/tests/$$package.out >> fullcover/fullcover.out ; \
done
@go tool cover -html=fullcover/fullcover.out -o fullcover/fullcover.html
@printf 'Full coverage on $(cpkg):'
@go tool cover -func fullcover/fullcover.out | tail -n -1 | awk '{$$1=""; $$2=""; sub(" ", " "); print}'
# whitepaper builds the whitepaper from whitepaper.tex. pdflatex has to be
# called twice because references will not update correctly the first time.
......@@ -108,4 +247,5 @@ whitepaper:
@pdflatex -output-directory=doc whitepaper.tex > /dev/null
pdflatex -output-directory=doc whitepaper.tex
.PHONY: all dependencies fmt REBUILD install release release-std xc clean test test-v test-long cover cover-integration cover-unit whitepaper
.PHONY: all fmt install release clean test test-v test-long cover whitepaper
Sia 0.4.7
=========
# [![Sia Logo](https://sia.tech/static/assets/svg/sia-green-logo.svg)](http://sia.tech)
[![Build Status](https://travis-ci.org/NebulousLabs/Sia.svg?branch=master)](https://travis-ci.org/NebulousLabs/Sia)
Binaries can be found at [our website](http://siacoin.com).
Sia is a new decentralized cloud storage platform aimed at giving users control
of their data. Data is split into hundreds of erasure coded pieces and
encrypted locally, and then each piece is uploaded to a separate host. A
blockchain is used to create cryptographic contracts ensuring that hosts will
only get paid if they actually store the data. Out of hundreds of hosts, only a
fraction are required to recover the original file.
Anybody can join the network as a host and get income from the storage they
contribute. This openness allows Sia to build and take advantage of a global
network of small datacenters. Combined with advanced algorithms for storing and
retrieving data, Sia is poised to be a highly competitive cloud storage
platform. More information about the technology can be found on our website and
in the 'doc' folder of the repo.
Sia is currently in beta. The currency was launched on June 7th, 2015, but the
storage platform itself remains in beta. Sia is ready for use with small sums
of money and non-critical files, but until the network has a more proven track
record, we advise against using it as a sole means of storing important data.
This release comes with 2 binaries, siad and siac. siad is a background
service, or "daemon," that runs the Sia protocol, and siac is a client that is
used to interact with siad. Siad exposes an API on 'localhost:9980' which can
be used to interact with the daemon. There is a front-end program called Sia-UI
which can be used to interact with the daemon in a more user-friendly way.
Documentation on the API can be found in doc/API.md.
Usage
-----
siad and siac are run via command prompt. On Windows, you can just double-
click siad.exe if you don't need to specify any command-line arguments.
Otherwise, navigate to the sia folder and click File->Open command prompt.
Then, start the siad service by entering `siad` and pressing Enter. The
command prompt may appear to freeze; this means siad is waiting for requests.
Windows users may see a warning from the Windows Firewall; be sure to check
both boxes ("Private networks" and "Public networks") and click "Allow
access." You can now run `siac` in a separate command prompt to interact with
siad. From here, you can send money, mine blocks, upload and download
files, and advertise yourself as a host.
Troubleshooting
---------------
- The client does not appear to join the network.
There should be at least one peer online for you to connect to, so if you
cannot connect, you may be experiencing connection issues. Ensure that you
are connected to the Internet. If you are confident that your connection is
okay, contact us! Our server may be experiencing problems.
You can also opt not to connect to join the network by passing the "-n" flag
to siad.
- I can't connect to more than 8 peers.
Once Sia has connected to 8 peers, it will stop trying to form new
connections, but it will still accept incoming connection requests (up to 128
total peers). However, if you are behind a firewall, you will not be able to
accept incoming connections. You must configure your firewall to allow Sia
connections by forwarding your ports. By default, Sia communicates on ports
9981 and 9982. The specific instructions for forwarding a port vary by
router. For more information, consult [this guide](http://portfoward.com).
In future versions, we will add support for UPnP, which may allow you to
skip this step if your router supports it.
- I'm 100% sure my ports are open, but Sia won't let me announce as a host.
siad tries to verify your connectivity by pinging your external IPv4 address.
This method is sufficient for most people, but for unusual setups it may
report false negatives. To override this check, you can "force" the
announcement by running `siac host announce [ip:port]`. You can determine
your external IP by running `siac gateway` or using a 3rd-party service.
- I mined a block, but I didn't receive any money.
There is a 144-block confirmation delay before you will receive siacoins from
mined blocks. If you still have not received the block reward after 144
blocks, it means your block did not made it into the blockchain.
- siad complains about "locks held too long."
This is debugging output, and should not occur during normal use. Please
contact us if this happens.
If your issue is not addressed above, you can get in touch with us personally:
slack: http://slackin.siacoin.com (ping taek or nemo)
email:
david@nebulouslabs.com
luke@nebulouslabs.com
Version Information
-------------------
- v0.4.0 introduces wallet seeds, which can be used to regenerate your wallet
using only a passphrase. To create a wallet, run `siac wallet init`. Be sure
to write down the passphrase somewhere safe! You will also need it to unlock
the wallet each time you start siad.
As a result, compatibility with the old wallet.dat files has been broken. To
transfer funds from v0.3.3.3 to v0.4.0, you must send them from the v0.3.3.3
client to a v0.4.0 address. Note that you can run both clients at once (from
different folders).
- v0.4.0 uses erasure-coding to enable durability, availability, and efficiency
when uploading and downloading files. However, the new upload and download
algorithms have not yet been perfected. Specifically, you may observe slower-
than-expected download speeds. These algorithms will be a top priority in
future releases. After all, they are crucial to the Sia platform!
- The format of the block database has changed, which means you will need to
redownload the entire blockchain. This may take anywhere from 10 minutes to
6 hours. Check http://explore.siacoin.com to see the current block height.
- Ports are now automatically forwarded using UPnP. If your router supports
UPnP, you no longer need to manually set up port forwarding. This should
improve the general health of the network. Note that for now, there is no
way to disable UPnP, but you can always turn it off in your router settings.
- Much compatibility with the v0.3.3.3 host and renter has been broken. The
.sia format has changed, the network protocol is different, etc. The bottom
line is, if you were hosting on v0.3.3.3, you should continue to do so until
your contracts expire. If you were renting, you should download those files
with the v0.3.3.3 client and reupload them on v0.4.0. We apologize for the
inconvenience. Fortunately, storage is cheap and the network is small.
Please tell us about any problems you run into, and any features you want! The
advantage of being a beta user is that your feedback will have a large impact
on what we do in the next few months. Thank you!
Version History
---------------
October 2015:
v0.4.7 (patch)
- Dropped support for v0.3.3.x
v0.4.6 (patch)
- Removed over-agressive consistency check
v0.4.5 (patch)
- Fixed last prominent bug in block database
- Closed some dangling resource handles
v0.4.4 (patch)
- Uploading is much more reliable
- Price estimations are more accurate
- Bumped filesize limit to 20 GB
v0.4.3 (patch)
- Block database is now faster and more stable
- Wallet no longer freezes when unlocked during IBD
- Optimized block encoding/decoding
September 2015:
v0.4.2 (patch)
- HostDB is now smarter
- Tweaked renter contract creation
v0.4.1 (patch)
- Added support for loading v0.3.3.x wallets
- Better pruning of dead nodes
- Improve database consistency
August 2015:
v0.4.0: Second stable currency release.
- Wallets are encrypted and generated from seed phrases
- Files are erasure-coded and transferred in parallel
- The blockchain is now fully on-disk
- Added UPnP support
June 2015:
v0.3.3.3 (patch)
- Host announcements can be "forced"
- Wallets can be merged
- Unresponsive addresses are pruned from the node list
v0.3.3.2 (patch)
- Siafunds can be loaded and sent
- Added block explorer
- Patched two critical security vulnerabilities
v0.3.3.1 (hotfix)
- Mining API sends headers instead of entire blocks
- Slashed default hosting price
v0.3.3: First stable currency release.
- Set release target
- Added progress bars to uploads
- Rigorous testing of consensus code
May 2015:
v0.3.2: Fourth open beta release.
- Switched encryption from block cipher to stream cipher
- Updates are now signed
- Added API calls to support external miners
v0.3.1: Third open beta release.
- Blocks are now stored on-disk in a database
- Files can be shared via .sia files or ASCII-encoded data
- RPCs are now multiplexed over one physical connection
March 2015:
v0.3.0: Second open beta release.
Jan 2015:
v0.2.0: First open beta release.
Dec 2014:
v0.1.0: Closed beta release.
The Sia project has moved to [GitHub](https://github.com/SiaFoundation).
# Security Policy
## Supported Versions
We recommend to only use the most recent minor release.
## Reporting a Vulnerability
**Please do not file a public ticket** mentioning the vulnerability.
To disclose a vulnerability in Sia email the developers and support team at hello@sia.tech
The following keys may be used to communicate sensitive information to developers.
You can import a key by running the following command with that individual’s fingerprint: `gpg --recv-keys "<fingerprint>"` Ensure that you put quotes around fingerprints containing spaces.
You can also import directly using the ASCII pubkeys from the [developer pubkeys folder](doc/developer-pubkeys).
| Name | Fingerprint | Public Key |
|------|-------------|-------------|
| Luke Champine | 99C2 18A7 FA3C C119 562B D6C5 A5C1 CE07 4CBF 1D60 | [public key file](doc/developer-pubkeys/luke-champine-pubkey.asc) |
| Steve Funk | DC06 2FB0 872C B88F 295C 59CD 1158 B926 BBD4 62BF | [public key file](doc/developer-pubkeys/steve-funk-pubkey.asc) |
| David Vorick | 7DB7 4718 3258 F448 64D2 D254 8323 331E 0787 6D0D | [public key file](doc/developer-pubkeys/david-vorick-pubkey.asc) |
## Disclosure Policy
Vulnerability details may be shared with third parties after the vulnerability
has been fixed and the program owner has provided permission to disclose or
after 120 days from submission, whichever is sooner.
### Scope
There is nothing considered out-of-scope for testers and researchers following
the rules outlined in this policy.
### Rewards
Nebulous has no formal reward policy and researchers should not expect a reward
for discovering a vulnerability. Nebulous is nonetheless grateful for all
legitimate discoveries of vulnerabilities, and is happy to acknowledge the
vulnerability and the researchers after a fix has been widely deployed.
### Official Communication Channel
All communication regarding vulnerabilities must be done though encrypted emails to hello@sia.tech
## Comments on this Policy
If you have suggestions on how this process could be improved please submit a merge request.
## Commitment to disclose.io Core Terms
Below are the disclose.io Core Terms which the Sia project is committed to upholding.
### Introduction
Security is core to our values, and we value the input of hackers acting in good
faith to help us maintain a high standard for the security and privacy for our
users. This includes encouraging responsible vulnerability research and
disclosure. This policy sets out our definition of good faith in the context of
finding and reporting vulnerabilities, as well as what you can expect from us in
return.
### Expectation
When working with us according to this policy, you can expect us to:
- Extend Safe Harbor for your vulnerability research that is related to this policy;
- Work with you to understand and validate your report, including a timely initial response to the submission;
- Work to remediate discovered vulnerabilities in a timely manner; and
- Recognize your contribution to improving our security if you are the first to report a unique vulnerability, and your report triggers a code or configuration change.
### Ground Rules
To encourage vulnerability research and to avoid any confusion between good-faith hacking and malicious attack, we ask that you:
- Play by the rules. This includes following this policy, as well as any other relevant agreements. If there is any inconsistency between this policy and any other relevant terms, the terms of this policy will prevail;
- Report any vulnerability you’ve discovered promptly;
- Avoid violating the privacy of others, disrupting our systems, destroying data, and/or harming user experience;
- Use only the Official Channels to discuss vulnerability information with us;
- Keep the details of any discovered vulnerabilities confidential until they are fixed, according to the Disclosure Policy;
- Perform testing only on in-scope systems, and respect systems and activities which are out-of-scope;
- If a vulnerability provides unintended access to data: Limit the amount of data you access to the minimum required for effectively demonstrating a Proof of Concept; and cease testing and submit a report immediately if you encounter any user data during testing, such as Personally Identifiable Information (PII), Personal Healthcare Information (PHI), credit card data, or proprietary information;
- You should only interact with test accounts you own or with explicit permission from the account holder; and
- Do not engage in extortion.
### Safe Harbor
When conducting vulnerability research according to this policy, we consider this research to be:
- Authorized in accordance with the Computer Fraud and Abuse Act (CFAA) (and/or similar state laws), and we will not initiate or support legal action against you for accidental, good faith violations of this policy;
- Exempt from the Digital Millennium Copyright Act (DMCA), and we will not bring a claim against you for circumvention of technology controls;
- Exempt from restrictions in our Terms & Conditions that would interfere with conducting security research, and we waive those restrictions on a limited basis for work done under this policy; and
- Lawful, helpful to the overall security of the Internet, and conducted in good faith.
You are expected, as always, to comply with all applicable laws.
If at any time you have concerns or are uncertain whether your security research is consistent with this policy, please submit a report through one of our Official Channels before going any further.
package api
import (
"encoding/json"
"fmt"
"net/http"
"strings"
"time"
"github.com/stretchr/graceful"
)
const (
apiTimeout = 5 * time.Second
)
// HttpGET is a utility function for making http get requests to sia with a whitelisted user-agent
func HttpGET(url string) (resp *http.Response, err error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Header.Add("User-Agent", "Sia-Agent")
return new(http.Client).Do(req)
}
// HttpPOST is a utility function for making post requests to sia with a whitelisted user-agent
func HttpPOST(url string, data string) (resp *http.Response, err error) {
req, err := http.NewRequest("POST", url, strings.NewReader(data))
if err != nil {
return nil, err
}
req.Header.Add("User-Agent", "Sia-Agent")
req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
return new(http.Client).Do(req)
}
// handleHTTPRequest is a wrapper function that handles all incoming calls to
// the API.
func handleHTTPRequest(mux *http.ServeMux, url string, handler http.HandlerFunc) {
mux.HandleFunc(url, func(w http.ResponseWriter, req *http.Request) {
if !strings.Contains(req.UserAgent(), "Sia-Agent") {
writeError(w, "Browser access disabled due to security vulnerability. Use Sia-UI or siac.", http.StatusBadRequest)
return
}
handler(w, req)
})
}
// initAPI determines which functions handle each API call.
func (srv *Server) initAPI() {
mux := http.NewServeMux()
// 404 Calls
handleHTTPRequest(mux, "/", srv.unrecognizedCallHandler)
// Daemon API Calls - Unfinished
handleHTTPRequest(mux, "/daemon/constants", srv.daemonConstantsHandler)
handleHTTPRequest(mux, "/daemon/stop", srv.daemonStopHandler)
handleHTTPRequest(mux, "/daemon/version", srv.daemonVersionHandler)
handleHTTPRequest(mux, "/daemon/updates/apply", srv.daemonUpdatesApplyHandler)
handleHTTPRequest(mux, "/daemon/updates/check", srv.daemonUpdatesCheckHandler)
// Consensus API Calls
if srv.cs != nil {
handleHTTPRequest(mux, "/consensus", srv.consensusHandler) // GET
handleHTTPRequest(mux, "/consensus/block", srv.consensusBlockHandler) // GET
}
// Gateway API Calls - Unfinished
if srv.gateway != nil {
handleHTTPRequest(mux, "/gateway/status", srv.gatewayStatusHandler)
handleHTTPRequest(mux, "/gateway/peers/add", srv.gatewayPeersAddHandler)
handleHTTPRequest(mux, "/gateway/peers/remove", srv.gatewayPeersRemoveHandler)
}
// Host API Calls - Unfinished
if srv.host != nil {
handleHTTPRequest(mux, "/host/announce", srv.hostAnnounceHandler)
handleHTTPRequest(mux, "/host/configure", srv.hostConfigureHandler)
handleHTTPRequest(mux, "/host/status", srv.hostStatusHandler)
}
// HostDB API Calls - Unfinished
if srv.hostdb != nil {
handleHTTPRequest(mux, "/hostdb/hosts/active", srv.hostdbHostsActiveHandler)
handleHTTPRequest(mux, "/hostdb/hosts/all", srv.hostdbHostsAllHandler)
}
// Miner API Calls - Unfinished
if srv.miner != nil {
handleHTTPRequest(mux, "/miner/start", srv.minerStartHandler)
handleHTTPRequest(mux, "/miner/status", srv.minerStatusHandler)
handleHTTPRequest(mux, "/miner/stop", srv.minerStopHandler)
handleHTTPRequest(mux, "/miner/headerforwork", srv.minerHeaderforworkHandler)
handleHTTPRequest(mux, "/miner/submitheader", srv.minerSubmitheaderHandler)
}
// Renter API Calls - Unfinished
if srv.renter != nil {
handleHTTPRequest(mux, "/renter/downloadqueue", srv.renterDownloadqueueHandler)
handleHTTPRequest(mux, "/renter/files/delete", srv.renterFilesDeleteHandler)
handleHTTPRequest(mux, "/renter/files/download", srv.renterFilesDownloadHandler)
handleHTTPRequest(mux, "/renter/files/list", srv.renterFilesListHandler)
handleHTTPRequest(mux, "/renter/files/load", srv.renterFilesLoadHandler)
handleHTTPRequest(mux, "/renter/files/loadascii", srv.renterFilesLoadAsciiHandler)
handleHTTPRequest(mux, "/renter/files/rename", srv.renterFilesRenameHandler)
handleHTTPRequest(mux, "/renter/files/share", srv.renterFilesShareHandler)
handleHTTPRequest(mux, "/renter/files/shareascii", srv.renterFilesShareAsciiHandler)
handleHTTPRequest(mux, "/renter/files/upload", srv.renterFilesUploadHandler)
handleHTTPRequest(mux, "/renter/status", srv.renterStatusHandler)
}
// TransactionPool API Calls - Unfinished
if srv.tpool != nil {
handleHTTPRequest(mux, "/transactionpool/transactions", srv.transactionpoolTransactionsHandler)
}
// Wallet API Calls
if srv.wallet != nil {
handleHTTPRequest(mux, "/wallet", srv.walletHandler) // GET
handleHTTPRequest(mux, "/wallet/address", srv.walletAddressHandler) // GET
handleHTTPRequest(mux, "/wallet/addresses", srv.walletAddressesHandler) // GET
handleHTTPRequest(mux, "/wallet/backup", srv.walletBackupHandler) // POST
handleHTTPRequest(mux, "/wallet/encrypt", srv.walletEncryptHandler) // POST - COMPATv0.4.0
handleHTTPRequest(mux, "/wallet/init", srv.walletInitHandler) // POST
handleHTTPRequest(mux, "/wallet/load/033x", srv.walletLoad033xHandler) // POST
handleHTTPRequest(mux, "/wallet/load/seed", srv.walletLoadSeedHandler) // POST
handleHTTPRequest(mux, "/wallet/load/siag", srv.walletLoadSiagHandler) // POST
handleHTTPRequest(mux, "/wallet/lock", srv.walletLockHandler) // POST
handleHTTPRequest(mux, "/wallet/seeds", srv.walletSeedsHandler) // GET
handleHTTPRequest(mux, "/wallet/siacoins", srv.walletSiacoinsHandler) // POST
handleHTTPRequest(mux, "/wallet/siafunds", srv.walletSiafundsHandler) // POST
handleHTTPRequest(mux, "/wallet/transaction/", srv.walletTransactionHandler) // $(id) GET
handleHTTPRequest(mux, "/wallet/transactions", srv.walletTransactionsHandler) // GET
handleHTTPRequest(mux, "/wallet/transactions/", srv.walletTransactionsHandler) // $(addr) GET
handleHTTPRequest(mux, "/wallet/unlock", srv.walletUnlockHandler) // POST
}
// BlockExplorer API Calls - Unfinished
if srv.exp != nil {
handleHTTPRequest(mux, "/explorer/status", srv.explorerStatusHandler)
handleHTTPRequest(mux, "/explorer/blockdata", srv.explorerBlockDataHandler)
handleHTTPRequest(mux, "/explorer/gethash", srv.explorerGetHashHandler)
}
// Create graceful HTTP server
srv.apiServer = &graceful.Server{
Timeout: apiTimeout,
Server: &http.Server{Handler: mux},
}
}
// unrecognizedCallHandler handles calls to unknown pages (404).
func (srv *Server) unrecognizedCallHandler(w http.ResponseWriter, req *http.Request) {
w.WriteHeader(http.StatusNotFound)
fmt.Fprintf(w, "404 - Refer to API.md")
}
// writeError an error to the API caller.
func writeError(w http.ResponseWriter, msg string, err int) {
http.Error(w, msg, err)
}
// writeJSON writes the object to the ResponseWriter. If the encoding fails, an
// error is written instead.
func writeJSON(w http.ResponseWriter, obj interface{}) {
if json.NewEncoder(w).Encode(obj) != nil {
http.Error(w, "Failed to encode response", http.StatusInternalServerError)
}
}
// writeSuccess writes the success json object ({"Success":true}) to the
// ResponseWriter
func writeSuccess(w http.ResponseWriter) {
writeJSON(w, struct{ Success bool }{true})
}
package api
import (
"fmt"
"net/http"
"github.com/NebulousLabs/Sia/types"
)
// ConsensusGET contains general information about the consensus set, with tags
// to support idiomatic json encodings.
type ConsensusGET struct {
Height types.BlockHeight `json:"height"`
CurrentBlock types.BlockID `json:"currentblock"`
Target types.Target `json:"target"`
}
// ConsensusBlockGET contains a block.
type ConsensusBlockGET struct {
Block types.Block `json:"block"`
}
// consensusHandlerGET handles a GET request to /consensus.
func (srv *Server) consensusHandlerGET(w http.ResponseWriter, req *http.Request) {
id := srv.mu.RLock()
defer srv.mu.RUnlock(id)
curblockID := srv.currentBlock.ID()
currentTarget, _ := srv.cs.ChildTarget(curblockID)
writeJSON(w, ConsensusGET{
Height: types.BlockHeight(srv.blockchainHeight),
CurrentBlock: srv.currentBlock.ID(),
Target: currentTarget,
})
}
// consensusHandler handles the API calls to /consensus.
func (srv *Server) consensusHandler(w http.ResponseWriter, req *http.Request) {
if req.Method == "" || req.Method == "GET" {
srv.consensusHandlerGET(w, req)
return
}
writeError(w, "unrecognized method when calling /consensus", http.StatusBadRequest)
}
// consensusBlockHandlerGET handles a GET request to /consensus/block.
func (srv *Server) consensusBlockHandlerGET(w http.ResponseWriter, req *http.Request) {
var height types.BlockHeight
_, err := fmt.Sscan(req.FormValue("height"), &height)
if err != nil {
writeError(w, err.Error(), http.StatusBadRequest)
return
}
block, exists := srv.cs.BlockAtHeight(height)
if !exists {
writeError(w, "no block found at given height", http.StatusBadRequest)
return
}
writeJSON(w, ConsensusBlockGET{
Block: block,
})
}
// consensusBlockHandler handles the API calls to /consensus/block.
func (srv *Server) consensusBlockHandler(w http.ResponseWriter, req *http.Request) {
if req.Method == "" || req.Method == "GET" {
srv.consensusBlockHandlerGET(w, req)
return
}
writeError(w, "unrecognized method when calling /consensus/block", http.StatusBadRequest)
}
package api
import (
"reflect"
"testing"
"github.com/NebulousLabs/Sia/types"
)
// TestIntegrationConsensusGet probes the GET call to /consensus.
func TestIntegrationConsensusGET(t *testing.T) {
if testing.Short() {
t.SkipNow()
}
st, err := createServerTester("TestIntegrationConsensusGET")
if err != nil {
t.Fatal(err)
}
var cg ConsensusGET
err = st.getAPI("/consensus", &cg)
if err != nil {
t.Fatal(err)
}
if cg.Height != 4 {
t.Error("wrong height returned in consensus GET call")
}
if cg.CurrentBlock != st.server.currentBlock.ID() {
t.Error("wrong block returned in consensus GET call")
}
expectedTarget := types.Target{128}
if cg.Target != expectedTarget {
t.Error("wrong target returned in consensus GET call")
}
}
// TestIntegrationConsensusBlockGET probes the GET call to /consensus/block.
func TestIntegrationConsensusBlockGET(t *testing.T) {
if testing.Short() {
t.SkipNow()
}
st, err := createServerTester("TestIntegrationConsensusBlockGET")
if err != nil {
t.Fatal(err)
}
var cbg ConsensusBlockGET
err = st.getAPI("/consensus/block?height=1", &cbg)
if err != nil {
t.Fatal(err)
}
block1, exists := st.cs.BlockAtHeight(1)
if !exists {
t.Fatal("unexpected dne")
}
if !reflect.DeepEqual(block1, cbg.Block) {
t.Fatal("blocks do not match")
}
// Sanity check - BlockAtHeight should be working.
gb := st.cs.GenesisBlock()
if block1.ParentID != gb.ID() {
t.Fatal("genesis block and block1 mismatch")
}
}
package api
import (
"errors"
"io/ioutil"
"math/big"
"net/http"
"path/filepath"
"runtime"
"strings"
"time"
"github.com/inconshreveable/go-update"
"github.com/kardianos/osext"
"github.com/NebulousLabs/Sia/build"
"github.com/NebulousLabs/Sia/types"
)
const (
developerKey = `-----BEGIN PUBLIC KEY-----
MIIEIjANBgkqhkiG9w0BAQEFAAOCBA8AMIIECgKCBAEAsoQHOEU6s/EqMDtw5HvA
YPTUaBgnviMFbG3bMsRqSCD8ug4XJYh+Ik6WP0xgq+OPDehPiaXK8ghAtBiW1EJK
mBRwlABXAzREZg8wRfG4l8Zj6ckAPJOgLn0jobXy6/SCQ+jZSWh4Y8DYr+LA3Mn3
EOga7Jvhpc3fTZ232GBGJ1BobuNfRfYmwxSphv+T4vzIA3JUjVfa8pYZGIjh5XbJ
5M8Lef0Xa9eqr6lYm5kQoOIXeOW56ImqI2BKg/I9NGw9phSPbwaFfy1V2kfHp5Xy
DtKnyj/O9zDi+qUKjoIivnEoV+3DkioHUWv7Fpf7yx/9cPyckwvaBsTd9Cfp4uBx
qJ5Qyv69VZQiD6DikNwgzjGbIjiLwfTObhInKZUoYl48yzgkR80ja5TW0SoidNvO
4WTbWcLolOl522VarTs7wlgbq0Ad7yrNVnHzo447v2iT20ILH2oeAcZqvpcvRmTl
U6uKoaVmBH3D3Y19dPluOjK53BrqfQ5L8RFli2wEJktPsi5fUTd4UI9BgnUieuDz
S7h/VH9bv9ZVvyjpu/uVjdvaikT3zbIy9J6wS6uE5qPLPhI4B9HgbrQ03muDGpql
gZrMiL3GdYrBiqpIbaWHfM0eMWEK3ZScUdtCgUXMMrkvaUJ4g9wEgbONFVVOMIV+
YubIuzBFqug6WyxN/EAM/6Fss832AwVPcYM0NDTVGVdVplLMdN8YNjrYuaPngBCG
e8QaTWtHzLujyBIkVdAHqfkRS65jp7JLLMx7jUA74/E/v+0cNew3Y1p2gt3iQH8t
w93xn9IPUfQympc4h3KerP/Yn6P/qAh68jQkOiMMS+VbCq/BOn8Q3GbR+8rQ8dmk
qVoGA7XrPQ6bymKBTghk2Ek+ZjxrpAoj0xYoYyzWf0kuxeOT8kAjlLLmfQ8pm75S
QHLqH49FyfeETIU02rkw2oMOX/EYdJzZukHuouwbpKSElpRx+xTnaSemMJo+U7oX
xVjma3Zynh9w12abnFWkZKtrxwXv7FCSzb0UZmMWUqWzCS03Rrlur21jp4q2Wl71
Vt92xe5YbC/jbh386F1e/qGq6p+D1AmBynIpp/HE6fPsc9LWgJDDkREZcp7hthGW
IdYPeP3CesFHnsZMueZRib0i7lNUkBSRneO1y/C9poNv1vOeTCNEE0jvhp/XOJuc
yCQtrUSNALsvm7F+bnwP2F7K34k7MOlOgnTGqCqW+9WwBcjR44B0HI+YERCcRmJ8
krBuVo9OBMV0cYBWpjo3UI9j3lHESCYhLnCz7SPap7C1yORc2ydJh+qjKqdLBHom
t+JydcdJLbIG+kb3jB9QIIu5A4TlSGlHV6ewtxIWLS1473jEkITiVTt0Y5k+VLfW
bwIDAQAB
-----END PUBLIC KEY-----`
)
// Updates work like this: each version is stored in a folder on a Linode
// server operated by the developers. The most recent version is stored in
// current/. The folder contains the files changed by the update, as well as a
// MANIFEST file that contains the version number and a file listing. To check
// for an update, we first read the version number from current/MANIFEST. If
// the version is newer, we download and apply the files listed in the update
// manifest.
var updateURL = "http://23.239.14.98/releases/" + runtime.GOOS + "_" + runtime.GOARCH
// SiaConstants is a struct listing all of the constants in use.
type SiaConstants struct {
BlockSizeLimit uint64
BlockFrequency types.BlockHeight
TargetWindow types.BlockHeight
MedianTimestampWindow uint64
FutureThreshold types.Timestamp
SiafundCount types.Currency
SiafundPortion *big.Rat
InitialCoinbase uint64
MinimumCoinbase uint64
MaturityDelay types.BlockHeight
GenesisTimestamp types.Timestamp
GenesisSiafundUnlockHash types.UnlockHash
GenesisClaimUnlockHash types.UnlockHash
RootTarget types.Target
RootDepth types.Target
MaxAdjustmentUp *big.Rat
MaxAdjustmentDown *big.Rat
SiacoinPrecision types.Currency
}
type UpdateInfo struct {
Available bool
Version string
}
// getHTTP is a helper function that returns the full response of an HTTP call
// to the update server.
func getHTTP(version, filename string) ([]byte, error) {
resp, err := http.Get(updateURL + "/" + version + "/" + filename)
if err != nil {
return nil, err
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
if resp.StatusCode != http.StatusOK {
return nil, errors.New(string(data))
}
return data, err
}
// fetchManifest requests and parses the update manifest. It returns the
// manifest (if available) as a slice of lines.
func fetchManifest(version string) (lines []string, err error) {
manifest, err := getHTTP(version, "MANIFEST")
if err != nil {
return
}
lines = strings.Split(strings.TrimSpace(string(manifest)), "\n")
if len(lines) == 0 {
err = errors.New("could not parse MANIFEST file")
}
return
}
// checkForUpdate checks a centralized server for a more recent version of
// Sia. If an update is available, it returns true, along with the newer
// version.
func checkForUpdate() (bool, string, error) {
manifest, err := fetchManifest("current")
if err != nil {
return false, "", err
}
version := manifest[0]
return build.VersionCmp(build.Version, version) < 0, version, nil
}
// applyUpdate downloads and applies an update.
func applyUpdate(version string) error {
manifest, err := fetchManifest(version)
if err != nil {
return err
}
// Get the executable directory.
binDir, err := osext.ExecutableFolder()
if err != nil {
return err
}
// Configure the update.
var opts update.Options
opts.SetPublicKeyPEM([]byte(developerKey))
// Perform updates as indicated by the manifest.
for _, file := range manifest[1:] {
// set update path
opts.TargetPath = filepath.Join(binDir, file)
// fetch the signature
opts.Signature, err = getHTTP(version, file+".sig")
if err != nil {
return err
}
// read update body
var resp *http.Response
resp, err = http.Get(updateURL + "/" + version + "/" + file)
if err != nil {
return err
}
err = update.Apply(resp.Body, opts)
resp.Body.Close()
if err != nil {
return err
}
}
return nil
}
// daemonStopHandler handles the API call to stop the daemon cleanly.
func (srv *Server) daemonStopHandler(w http.ResponseWriter, req *http.Request) {
// can't write after we stop the server, so lie a bit.
writeSuccess(w)
// send stop signal
srv.apiServer.Stop(time.Second)
}
// daemonVersionHandler handles the API call that requests the daemon's version.
func (srv *Server) daemonVersionHandler(w http.ResponseWriter, req *http.Request) {
writeJSON(w, build.Version)
}
// daemonUpdatesCheckHandler handles the API call to check for daemon updates.
func (srv *Server) daemonUpdatesCheckHandler(w http.ResponseWriter, req *http.Request) {
available, version, err := checkForUpdate()
if err != nil {
writeError(w, err.Error(), http.StatusInternalServerError)
return
}
writeJSON(w, UpdateInfo{available, version})
}
// daemonUpdatesApplyHandler handles the API call to apply daemon updates.
func (srv *Server) daemonUpdatesApplyHandler(w http.ResponseWriter, req *http.Request) {
err := applyUpdate(req.FormValue("version"))
if err != nil {
writeError(w, err.Error(), http.StatusInternalServerError)
return
}
writeSuccess(w)
}
// debugConstantsHandler prints a json file containing all of the constants.
func (srv *Server) daemonConstantsHandler(w http.ResponseWriter, req *http.Request) {
sc := SiaConstants{
GenesisTimestamp: types.GenesisTimestamp,
BlockSizeLimit: types.BlockSizeLimit,
BlockFrequency: types.BlockFrequency,
TargetWindow: types.TargetWindow,
MedianTimestampWindow: types.MedianTimestampWindow,
FutureThreshold: types.FutureThreshold,
SiafundCount: types.SiafundCount,
MaturityDelay: types.MaturityDelay,
SiafundPortion: types.SiafundPortion,
InitialCoinbase: types.InitialCoinbase,
MinimumCoinbase: types.MinimumCoinbase,
SiacoinPrecision: types.SiacoinPrecision,
RootTarget: types.RootTarget,
RootDepth: types.RootDepth,
MaxAdjustmentUp: types.MaxAdjustmentUp,
MaxAdjustmentDown: types.MaxAdjustmentDown,
}
writeJSON(w, sc)
}