.gitlab-ci.yml 7.54 KB
Newer Older
1
image: debian:buster
2

3
stages:
4
 - deploy
5
 - production
6
7

variables:
8
9
  LC_ALL: C.UTF-8
  DEBIAN_FRONTEND: noninteractive
10
11
  OUT_DIR: build

12
13
# This template needs to be in text block format since gitlab-runner
# exec cannot handling templates in list format.
14
.apt-template: &apt-template |
15
16
  set -x
  set -e
17
18
19
20
21
22
23
  echo Etc/UTC > /etc/timezone
  echo 'quiet "1";' \
       'APT::Install-Recommends "0";' \
       'APT::Install-Suggests "0";' \
       'APT::Acquire::Retries "20";' \
       'APT::Get::Assume-Yes "true";' \
       'Dpkg::Use-Pty "0";' \
24
      > /etc/apt/apt.conf.d/99gitlab
25
  echo "deb http://deb.debian.org/debian/ buster-backports main" >> /etc/apt/sources.list
26
  printf "Package\x3a po4a ruby-git ruby-jekyll-include-cache ruby-jekyll-last-modified-at ruby-jekyll-paginate-v2 ruby-jekyll-redirect-from ruby-jekyll-sitemap ruby-loofah ruby-nokogiri ruby-rchardet ruby-rouge ruby-zip\nPin\x3a release a=buster-backports\nPin-Priority\x3a 500\n" > /etc/apt/preferences.d/debian-buster-backports.pref
27
28
  apt-get update
  apt-get dist-upgrade
29

30

31
# Common steps required for each type of "Build" (f-droid.org, GitLab Pages, feature branches)
32
33
34
#
# This template needs to be in text block format since gitlab-runner
# exec cannot handling templates in list format.
35
.setup_for_jekyll: &setup_for_jekyll |
36
37
  set -x
  set -e
38
  apt-get install --install-recommends git libunicode-linebreak-perl po4a
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
  apt-get install \
    gettext \
    linkchecker \
    python3-babel \
    rsync \
    rubocop \
    ruby-bundler \
    ruby-git \
    ruby-jekyll-include-cache \
    ruby-jekyll-paginate-v2 \
    ruby-jekyll-redirect-from \
    ruby-json \
    ruby-loofah \
    ruby-nokogiri \
    ruby-rchardet \
    ruby-rouge \
    ruby-rspec \
    ruby-zip \
    unzip
  bundle install --local --verbose
59
  ./tools/i18n.sh
60
61


flowed's avatar
flowed committed
62
# This is a manual task for building in preparation to deploy to
63
64
65
# https://f-droid.org. The intention is for it to be run locally using
# `gitlab-runner` each time a tag is found that is signed by a key in
# the whitelist keyring.  Invoke like so:
66
#
67
68
#  gitlab-runner exec docker f-droid.org --pre-build-script ./prepare-for-deploy.py \
#    --docker-volumes "/root/deploy-whitelist-keyring.gpg:/root/.gnupg/pubring.gpg:ro" \
69
#    --docker-volumes `pwd`/_site:/builds/output
70
#
71
72
73
# And when it is finished, you should have a directory in _site/build/
# which includes the entire static site ready to be deployed to
# https://f-droid.org.
74
f-droid.org:
75
  stage: production
76
77
78
  only:
    - tags@fdroid/fdroidserver
    - master@fdroid/fdroidserver
79
80
  when: manual
  script:
81
   - '[ ! -d /builds/output ] && echo "ERROR: /builds/output is not mounted inside docker!" && exit 1'
82
   - *apt-template
83
   - *setup_for_jekyll
84
85
86
87
88
89
   - sed -Ei
         -e "s,^(url\x3a).*,\1 https://f-droid.org,"
         -e 's,^(baseurl\x3a).*,\1 "",'
         _config.yml
   - echo "Jekyll config used for CI:" && cat _config.yml
   - jekyll build -d $OUT_DIR --trace
90
   - ./tools/prepare-multi-lang.sh $OUT_DIR
91
   - ./tools/deploy-external-assets.sh $OUT_DIR
92
   - rsync -ax --delete $OUT_DIR /builds/output/
93

94

95
96
pages:
  stage: deploy
97
98
  except:
    - triggers
99
  artifacts:
100
101
    paths:
      - public
102
    expire_in: 1w
103
    when: always
104
  script:
105
106
107
   - (find _*/ -type f | grep -Ev '\.(html|json|md|rb|scss|xml|yaml)$')
         && (echo "ERROR The above files have a bad or missing file extension"; exit 1)

108
109
   - *apt-template
   - apt-get install curl python3-yaml
110
   - ./tools/trigger-spellcheckbot
111
112
113
   # use the 'gitlab ci' subset of languages
   - sed -i 's,^languages:,ignored_languages:,' _config.yml
   - sed -i 's,^gitlab_ci_languages:,languages:,' _config.yml
114
   - *setup_for_jekyll
115
   - ./tools/check-format-strings.py
116
   - ./tools/check-page-links.py
117
   - ./tools/check-yaml-front-matter.py
118
   - ./tools/check-do-not-translate
119
   - ./tools/check-markdown-headers-are-localizable.py
120
121
   # This is where GitLab pages will deploy to by default (e.g. "https://fdroid.gitlab.io/fdroid-website")
   # so we need to make sure that the Jekyll configuration understands this.
122
123
124
125
126
   - sed -Ei
         -e "s,^(url\x3a).*,\1 'https://$CI_PROJECT_NAMESPACE.gitlab.io',"
         -e "s,^(baseurl\x3a).*,\1 '/$CI_PROJECT_NAME',"
         _config.yml
   - echo "Jekyll config used for CI:" && cat _config.yml
127
   - jekyll build -d public --trace --future
128
   - cp public/robots.txt.noindex public/robots.txt
129
   - ./tools/prepare-multi-lang.sh public --no-type-maps
130
131
   - mkdir linkchecker/
   - ln -s ../public linkchecker/$CI_PROJECT_NAME
132
   - ruby -run -e httpd linkchecker/ -p 4000 > /dev/null 2>&1 &
133
   - linkchecker http://localhost:4000/$CI_PROJECT_NAME --config=.linkcheckerrc
134
135


136
137
# check rsync mirrors still available
rsync mirrors:
138
139
  image: debian:bullseye-slim
  stage: deploy
140
141
142
143
144
  only:
    - master@fdroid/fdroidserver
  script:
    - apt-get update
    - apt-get -qy install rsync
145
146
147
148
    - for f in `sed 's,[^:]*:,,' _data/rsync_mirrors.yaml`; do
          rsync --list-only rsync -axv ${f}::fdroid \
          || rsync --list-only rsync -axv ${f}::fdroid \
          || rsync --list-only rsync -axv ${f}::fdroid;
149
150
151
      done


152
153
spellcheckbot:
  image: node:buster
154
155
  stage: deploy
  allow_failure: true
156
157
  only:
    - triggers
158
  script:
159
160
161
162
    - test -n ${FROM_CI_PROJECT_URL}
    - test -n ${FROM_CI_COMMIT_SHA}
    - git fetch ${FROM_CI_PROJECT_URL} ${FROM_CI_COMMIT_SHA}
    - for f in `git diff --name-only --diff-filter=d HEAD...${FROM_CI_COMMIT_SHA}`; do
163
164
165
          export CHANGED="$CHANGED `echo $f | grep '\.md$' || true`";
      done
    - if [ -z "`echo $CHANGED | sed 's,\s*,,g'`" ]; then
166
167
          echo "No markdown files changed";
          exit 0;
168
169
      else
          echo "Spellchecking $CHANGED";
170
          git checkout --force ${FROM_CI_COMMIT_SHA};
171
      fi
172
    - apt-get -qy update
173
    - apt-get -qy install --no-install-recommends --allow-unauthenticated python3-gitlab python3-requests
174
175
    - ./tools/fetch-spelling-words.py >> .spelling
    - npm i markdown-spellcheck -g
176
177
178
    - (mdspell --report --en-gb --ignore-numbers --ignore-acronyms $CHANGED > output.txt 2>&1)
        || ./tools/spellcheckbot.py
    - cat output.txt
179

180
181
182
183
184
185
186
187
188

# Download and verify that the FDroid.apk is signed by the right PGP
# key.  The only time that F-Droid's signed metadata does not verify
# the APK is the initial download and install of F-Droid itself.  An
# attacker could replace the FDroid.apk and PGP signature on the
# website. The gpg key model is to trust only the key that is included
# in this script, so there is a test to check that it is starting with
# an empty keyring.

189
check_fdroid_apk_bot:
190
191
192
  stage: deploy
  only:
    - schedules
193
    - master@fdroid/fdroidserver
194
195
  image: alpine:3.5
  variables:
196
197
    apk: F-Droid.apk
    asc: F-Droid.apk.asc
198
    curl: "curl --user-agent F-Droid --retry 99"
199
    fingerprint: 37D2C98789D8311948394E3E41E7044E1DBA2E89
200
    pip: pip3 --timeout 100 --retries 10
201
202
203
204
205
206
207
  artifacts:
    name: "$apk-failed-${CI_JOB_ID}"
    paths:
      - $apk
      - $asc
    expire_in: 180 days
    when: on_failure
208
209
  script:
    - apk add --no-cache gnupg curl
210
    - "! (gpg --list-keys | grep pub)"
211
212
213
214
    - $curl https://f-droid.org/admin@f-droid.org.gpg | gpg --import || true
    - gpg --recv-key $fingerprint || true
    - gpg --keyserver https://keyserver.ubuntu.com --recv-key $fingerprint || true
    - gpg --list-key $fingerprint
215
216
217
    - echo "${fingerprint}:6:" | gpg --import-ownertrust
    - $curl https://f-droid.org/$apk > $apk
    - $curl https://f-droid.org/$asc > $asc
218
    - ls -l $apk $asc
219
    - sha256sum $apk
220
221
222
223
    - gpg --batch --trust-model always --verify $asc $apk || (
          apk add --no-cache python3;
          python3 -m ensurepip;
          $pip install python-gitlab;
224
          ./tools/run-check-fdroid-apk-bot.py;
225
      )