Commit 26d7f9e1 authored by Hans-Christoph Steiner's avatar Hans-Christoph Steiner

switch to manually choosing supported languages

Since the website is built from 4 sources, and we currently cannot support
many languages, the supported languages need to be manually chosen.  The
way this is currently, it takes 6 hours to run on an 8gig / 2 core VM.

refs #128
parent 9ac63251
# This file is automatically generated by tools/update_langs.sh
SUPPORTED_LANGS="en de es es_AR fr sc tr zh_Hans zh_Hant"
......@@ -25,10 +25,14 @@ feed:
path: feed
fdroid-repo: https://f-droid.org/repo
# The "languages" entry is automatically overridden by the tools/update_langs.sh script when
# new translations become available.
languages: [ "en", "de", "es", "es_AR", "fr", "sc", "tr", "zh_Hans", "zh_Hant" ]
default_lang: "_" # Not specifying a language here forces polyglot to generate a en/ subdirectory too (with correct links). See https://github.com/untra/polyglot/issues/60.
# Languages are manually chosen by looking at languages that are fully
# translated across the whole project. Check with this script:
# ./tools/check-language-status.sh
languages: [ "bo", "en", "de", "es", "fr", "tr", "zh_Hans", "zh_Hant" ]
# Specifying the default language as "_" forces polyglot to generate a
# en/ subdirectory too (with correct links). See
# https://github.com/untra/polyglot/issues/60
default_lang: "_"
exclude_from_localization:
- wp-content
- assets
......
#!/bin/sh
# The CSV from Weblate contains the following header row:
# name,code,total,translated,translated_percent,total_words,translated_words,failing,failing_percent,fuzzy,fuzzy_percent,url_translate,url,last_change,last_author
# where the columns of interest are $5 (translated_percent) and $2 (code).
for f in f-droid fdroiddata repomaker repomaker-javascript website website-docs website-posts; do
echo $f `curl --silent https://hosted.weblate.org/exports/stats/f-droid/$f/\?format\=csv | tail -n +2 | awk -F ',' '{ if ($5 > 95.0) print $2 }' | sort -u`
done
......@@ -106,7 +106,6 @@ function po2md {
#generate_md_files _docs _docs
#generate_md_files _posts _posts
generate_md_files _pages _docs
${DIR_SRC}/tools/update_langs.sh
rm -r "${DIR_BUILD}"
}
......
......@@ -90,9 +90,8 @@ TXT
}
if [ ! -f .supported-langs ]; then
echo "Must be run from the same directory as the .supported-langs file (the Jekyll source dir)."
echo "If that file doesn't exist, then you can generate it with ./tools/update_langs.sh"
if [ ! -f _config.yml ]; then
echo "Must be run from the same directory as _config.yml (the Jekyll source dir)."
exit 1
fi
......@@ -105,7 +104,7 @@ if [[ $# == 0 || ! -d $1/en ]]; then
fi
# This populates the SUPPORTED_LANGS variable.
source .supported-langs
SUPPORTED_LANGS=`ruby -ryaml -e 'puts YAML.load_file("_config.yml")["languages"]'`
# This populates the WEBLATE_LANGS variable.
source tools/weblate-supported-langs.sh
......
#!/usr/bin/env bash
# Fail on errors.
set -eu -o pipefail
DIR=`dirname $0`
JEKYLL_DIR=${DIR}/../
# The default behaviour is to only include languages where `_data/LANG/strings.json` is 100% translated.
# This can be overriden by "--partial", to include any langugae with at least a single translation.
if [[ $# == 1 && $1 == "--partial" ]]; then
INCLUDE_PARTIAL=true
else
INCLUDE_PARTIAL=false
fi
#
# For each directory `_data/[LANG]/ which contains a `strings.json` file, we take LANG and add it to our config file so
# that Jekyll knows to generate a site for that language.
#
function update_langs_in_config {
LANGS_FOR_YAML='"en"'
LANGS_FOR_SH='en'
if [ ${INCLUDE_PARTIAL} == true ]; then
echo "Including only partially translated languages"
else
echo "Consulting Weblate to figure out 100% transtlated languages"
FULLY_TRANSLATED_LANGS=`get_fully_translated_langs_from_weblate`
fi
for I18N_DIR in `find ${JEKYLL_DIR}/_data/*/ -name 'strings.json' -printf '%h\n' | sort -u`; do
# Check if string contains $LANG already or not: http://stackoverflow.com/a/229606/2391921
LANG=`basename ${I18N_DIR}`
# Check the diff between original strings.json and translated one. If no difference, then bail.
DIFF2=`diff --ignore-all-space ${I18N_DIR}/strings.json ${JEKYLL_DIR}/_data/strings.json | wc -l || true`
if [ ${DIFF2} == "0" ]; then
echo "Ignoring completely untranslated ${LANG}/strings.json"
continue;
fi
if [ ${INCLUDE_PARTIAL} == false ]; then
IN_WEBLATE=`echo ${FULLY_TRANSLATED_LANGS} | grep --only-matching "\<$LANG\>" | wc -l || true`
if [ ${IN_WEBLATE} == "0" ]; then
echo "Ignoring not-yet-fully-translated ${LANG}/strings.json"
continue;
fi
fi
if [[ ${LANGS_FOR_YAML} != *"$LANG"* ]]; then
LANGS_FOR_YAML="$LANGS_FOR_YAML, \"$LANG\""
LANGS_FOR_SH="$LANGS_FOR_SH $LANG"
fi
done
if [ ${INCLUDE_PARTIAL} == true ]; then
for PO in `ls ${JEKYLL_DIR}/po/*.*.po`; do
PO_FILE=`basename ${PO}`
LANG=`echo ${PO_FILE} | sed -e "s|.*\.\(.*\)\.po|\1|"`
# Check the number of messages in the .po file which have been translated. If none, then bail.
TRANSLATED=`msgattrib --translated ${JEKYLL_DIR}/po/${PO_FILE} | wc -l`
if [ ${TRANSLATED} == "0" ]; then
echo "Ignoring completely untranslated $PO_FILE"
continue;
fi
if [[ ${LANGS_FOR_YAML} != *"$LANG"* ]]; then
LANGS_FOR_YAML="$LANGS_FOR_YAML, \"$LANG\""
LANGS_FOR_SH="$LANGS_FOR_SH $LANG"
fi
done
fi
echo "Updating languages to: ${LANGS_FOR_YAML}"
sed -i "s/^languages: \[.*\]/languages: [ ${LANGS_FOR_YAML} ]/" ${JEKYLL_DIR}/_config.yml
# This is used by other script which need to know which languages are available.
cat<<SH > ${JEKYLL_DIR}/.supported-langs
# This file is automatically generated by tools/update_langs.sh
SUPPORTED_LANGS="${LANGS_FOR_SH}"
SH
}
function get_fully_translated_langs_from_weblate {
# The CSV from Weblate contains the following header row:
# name,code,total,translated,translated_percent,total_words,translated_words,failing,failing_percent,fuzzy,fuzzy_percent,url_translate,url,last_change,last_author
# where the columns of interest are $5 (translated_percent) and $2 (code).
curl --silent https://hosted.weblate.org/exports/stats/f-droid/website/\?format\=csv | \
tail -n +2 | \
awk -F ',' '{ if ($5 == "100.0") print $2 }'
}
update_langs_in_config
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment