Commit b3a65719 authored by  jpegxl-bot's avatar jpegxl-bot 🤖

Update JPEG-XL with latest changes.

This includes all changes up to 2020-02-06 12:34:43 +0100.
parent ff093712
# Copyright (c) the JPEG XL Project
#
# Use of this source code is governed by an MIT-style
# license that can be found in the LICENSE file or at
# https://opensource.org/licenses/MIT.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# We define only two stages for development. The "build" stage only compiles the
# code and doesn't run any target code. The "test" stage runs the code compiled
......@@ -22,7 +30,7 @@ variables:
# A template for running in the generic cloud builders. These are tagged with
# "linux" and run on shared VMs.
.linux_builder_template: &linux_builder_template
image: &jpegxl-builder gcr.io/jpegxl/[email protected]:b4744eed15a00e88d6faf54798c8d8bca8b58a581bfdda374cfa8c668ec6c1b2
image: &jpegxl-builder gcr.io/jpegxl/[email protected]:e97bb79499eb4f9af4186388a450996b2c27caa6cc873f4175947ae398192bbb
tags:
- linux
# By default all the workflows run on master and on request. This can be
......@@ -85,6 +93,7 @@ variables:
# Common artifacts for the build stage
.default_build_paths: &default_build_paths
- build/libjpegxl.so*
- build/tools/cbrunsli
- build/tools/cjpegxl
- build/tools/djpegxl
- build/tools/benchmark_xl
......@@ -463,7 +472,7 @@ test:fast_benchmark.msan:
# This template runs on actual aarch64 hardware.
.benchmark_aarch64_template: &benchmark_aarch64_template
<<: *linux_builder_template
image: gcr.io/jpegxl/[email protected]:4ff7fe5e3e1d19a8c4604b72a41916173d83ec95d335f7fabd599ad92c438df2
image: gcr.io/jpegxl/[email protected]:6d218aaf961c3e0591a7a0852cb8d30524811c8f9ed7327bb886e6d7e8294c1c
stage: test
tags:
- aarch64
......@@ -532,6 +541,11 @@ build:tidy:all:
build:ems:all:
<<: *linux_x86_64_template
stage: build
only:
- master
- merge_requests
- schedules
- tags
variables:
CC: emcc
CXX: em++
......
......@@ -10,9 +10,6 @@
[submodule "third_party/googletest"]
path = third_party/googletest
url = https://github.com/google/googletest
[submodule "third_party/gperftools"]
path = third_party/gperftools
url = https://github.com/gperftools/gperftools
[submodule "third_party/brunsli"]
path = third_party/brunsli
url = https://github.com/google/brunsli.git
......
......@@ -48,7 +48,7 @@ if(JPEGXL_EMSCRIPTEN)
message("-- Compiler is EMSCRIPTEN.")
set(JPEGXL_NOT_EMSCRIPTEN false)
set(CMAKE_STATIC_LIBRARY_SUFFIX ".bc")
set(CMAKE_EXECUTABLE_SUFFIX ".html")
set(CMAKE_EXECUTABLE_SUFFIX ".js")
add_definitions(-DPROFILER_ENABLED=0)
else()
message("-- Compiler is not EMSCRIPTEN.")
......@@ -56,15 +56,13 @@ else()
endif()
### Project build options:
if(NOT APPLE AND NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64"
AND NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm")
# Enabled by default except on arm64 and Apple builds.
if(NOT APPLE AND NOT WIN32 AND NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64"
AND NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm" AND NOT JPEGXL_EMSCRIPTEN)
# Enabled by default except on arm64, Windows and Apple builds.
set(ENABLE_FUZZERS_DEFAULT true)
endif()
if(NOT APPLE AND NOT JPEGXL_EMSCRIPTEN
AND NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64"
AND NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm")
# Enabled by default except on arm64 and Apple builds.
if(NOT APPLE AND NOT WIN32 AND NOT JPEGXL_EMSCRIPTEN)
# Enabled by default except on Windows and Apple builds.
set(ENABLE_TCMALLOC_DEFAULT true)
endif()
......@@ -136,14 +134,14 @@ else()
endif()
# CPU flags
if(${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86_64")
if(${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86_64|AMD64" AND NOT MSVC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \
-mavx2 \
-mfma \
-Xclang -mprefer-vector-width=128 \
-Xclang -target-cpu -Xclang haswell \
-Xclang -target-feature -Xclang +avx2")
elseif(${CMAKE_SYSTEM_PROCESSOR} MATCHES "AMD64")
elseif(${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86_64|AMD64" AND MSVC)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2")
elseif(${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \
......@@ -177,6 +175,7 @@ set(CMAKE_CXX_STANDARD 11)
set(CMAKE_CXX_EXTENSIONS OFF)
set(CMAKE_CXX_STANDARD_REQUIRED YES)
find_package(PkgConfig)
add_subdirectory(third_party)
set(THREADS_PREFER_PTHREAD_FLAG YES)
......@@ -190,8 +189,6 @@ include(CTest)
# use in our project (library, tests and tools): JPEGXL_INTERNAL_FLAGS.
include(jpegxl.cmake)
if (NOT "${JPEGXL_EMSCRIPTEN}")
# Other libraries outside the core jpegxl library.
include(jpegxl_extras.cmake)
include(jpegxl_threads.cmake)
......@@ -211,14 +208,13 @@ include(GoogleTest)
# Tests for the jpegxl library.
include(jpegxltests.cmake)
else() # JPEGXL_EMSCRIPTEN
if ("${JPEGXL_EMSCRIPTEN}")
# WASM API facade.
add_executable(jpegxl_emcc jxl/jpegxl_emcc.cc)
target_link_libraries(jpegxl_emcc jpegxl-static)
set_target_properties(jpegxl_emcc PROPERTIES LINK_FLAGS "\
-O3\
-s TOTAL_MEMORY=256MB\
-s EXPORTED_FUNCTIONS='[\
\"_decompress\",\
\"_free\",\
......
......@@ -17,17 +17,17 @@ version of XCode is installed.
Installing (actually, building) `clang` might take a couple hours.
```shell
```bash
brew install llvm
```
```shell
```bash
brew install clang-format coreutils cmake giflib libjpeg ninja parallel
```
If `git-clang-format` command is not accessible, run
```shell
```bash
brew link --overwrite clang-format
```
......
......@@ -13,71 +13,139 @@ future decoders being able to decode the output of a current encoder.
## Checking out the code
This repository uses git submodules to handle some third party dependencies
under `third_party/`. To check out these dependencies as well clone the
under `third_party/`. To also check out these dependencies, clone the
repository with `--recursive`:
```shell
git clone [email protected]:wg1/jpeg-xl.git --recursive
```bash
git clone https://gitlab.com/wg1/jpeg-xl.git --recursive
```
If you didn't check out recursively, and after any update run the following
command to check out the git submodules.
If you didn't check out with `--recursive`, or any of the third party
dependencies have changed, run the following command:
```shell
```bash
git submodule update --init --recursive
```
## Minimum build dependencies
## Building
Apart from the dependencies in third_party, some of the tools use external
dependencies that need to be installed in your system first. For a Debian/Ubuntu
based Linux distribution install:
To avoid system incompatibilities, we **strongly recommend** using Docker to
build and test the software, as explained in the
[step by step guide](doc/developing_in_docker.md).
```shell
sudo apt install cmake clang-6.0 g++-8 qtbase5-dev libqt5x11extras5-dev \
extra-cmake-modules libgif-dev libjpeg-dev ninja-build
```
For experienced developers on an up to date Debian-based Linux system, we
also provide [Debian build instructions](doc/building_in_debian.md). If you
encounter any difficulties, please use Docker instead.
For developing changes in JPEG XL, take a look at the
[Building and Testing changes](doc/building_and_testing.md) guide.
The resulting binaries are in the `build` directory and its subdirectories.
Make sure your default "clang" compiler is at least version 6 running
## CPU requirements
```bash
clang --version
```
When running on Intel/AMD CPUs the software **currently requires AVX2** support
(introduced by Intel in 2013 and AMD in 2015). If not supported by the CPU or
VM, the software will print "CPU does not support all enabled targets =>
exiting". You can either run on a more recent CPU/VM, or instruct the software
to only use/require SSE4 by uncommenting the line near the bottom of
third_party/highway/hwy/target.h:
`// #define HWY_DISABLE_AVX2`
## Basic encoder/decoder
If it still shows an old version despite having a clang-6.0 installed, you need
to update the default clang compiler. In Debian-based systems run:
`build/tools/cjpegxl input.png output.jxl` encodes to JPEG XL with default
options.
Here and in general, the JPEG XL tools are able to read/write the following
image formats: .exr, .gif, .jpeg/.jpg, .pfm, .pgm/.ppm, .pgx, .png.
`build/tools/djpegxl output.jxl output.png` decodes JPEG XL to other formats.
## Benchmarking
We recommend `build/tools/benchmark_xl` as a convenient method for reading
images or image sequences, encoding them using various codecs (jpeg jxl png
webp), decoding the result, and computing objective quality metrics. An example
invocation is:
```bash
sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-6.0 100
sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 100
build/tools/benchmark_xl --input "/path/*.png" --codec jxl:wombat:d1,jxl:cheetah:d2
```
## Building
The project builds with cmake. We currently support Linux (tested with Debian).
To build the "release" version, you can use the following helper command:
Multiple comma-separated codecs are allowed. The characters after : are
parameters for the codec, separated by colons, in this case specifying maximum
target psychovisual distances of 1 and 2 (higher implies lower quality) and
the encoder effort (see below). Other common parameters are `r0.5` (target
bitrate 0.5 bits per pixel) and `q92` (quality 92, on a scale of 0-100, where
higher is better). The `jxl` codec supports the following additional parameters:
Speed: `falcon`, `cheetah`, `hare`, `wombat`, `squirrel`, `kitten`, `tortoise`
control the encoder effort in ascending order.
* `falcon` disables all of the following tools.
* `cheetah` enables coefficient reordering, context clustering, and heuristics
for selecting DCT sizes and quantization steps.
* `hare` enables Gaborish filtering, chroma from luma, and an initial estimate
of quantization steps.
* `wombat` enables error diffusion quantization and full DCT size selection
heuristics.
* `squirrel` enables dots, patches, and spline detection, and full context
clustering.
* `kitten` (default) enables the iterative adaptive quantization search.
* `tortoise` enables a more thorough adaptive quantization search.
Mode: JPEG XL has several modes for various types of content. The default mode
is suitable for photographic material. One of the following alternatives may be
selected:
* `mg` activates modular mode (useful for non-photographic images such as
screen content).
* `bg` activates lossless JPEG reconstruction with parallel decoding (the
input must have been a JPEG file).
* `b:file` activates lossless JPEG reconstruction with more compact encodings,
but without the option of parallel decoding.
Other arguments to benchmark_xl include:
* `save_compressed`: save codestreams to `output_dir`.
* `save_decompressed`: save decompressed outputs to `output_dir`.
* `output_extension`: selects the format used to output decoded images.
* `num_threads`: number of codec instances that will independently
encode/decode images, or 0.
* `inner_threads`: how many threads each instance should use for parallel
encoding/decoding, or 0.
* `encode_reps`/`decode_reps`: how many times to repeat encoding/decoding
each image, for more consistent measurements (we recommend 10).
The benchmark output begins with a header:
```shell
./ci.sh release
```
Compr Input Compr Compr Compr Decomp Butteraugli
Method Pixels Size BPP # MP/s MP/s Distance Error p norm BPP*pnorm Errors
```
This will build and test the project, and leave the binaries in the `build/`
directory. Check out the `tools` subdirectory for command-line tools that
interact with the library. You can set the environment variable SKIP_TEST=1 to
skip the test stage.
`ComprMethod` lists each each comma-separated codec. `InputPixels` is the number
of pixels in the input image. `ComprSize` is the codestream size in bytes and
`ComprBPP` the bitrate. `Compr MP/s` and `Decomp MP/s` are the
compress/decompress throughput, in units of Megapixels/second.
`Butteraugli Distance` indicates the maximum psychovisual error in the decoded
image (larger is worse). `Error p norm` is a similar summary of the psychovisual
error, but closer to an average, giving less weight to small low-quality
regions. `BPP*pnorm` is the product of `ComprBPP` and `Error p norm`, which is a
figure of merit for the codec (lower is better). `Errors` is nonzero if errors
occurred while loading or encoding/decoding the image.
There are other build versions with more debug information useful when
developing. You can read more about it in the
[Building and Testing changes](doc/building_and_testing.md) guide.
## Additional documentation
## Documentation
### Codec description
* [Developing in GitLab](doc/developing_in_gitlab.md)
* [XL Overview](doc/xl_overview.md)
* [Building and Testing changes](doc/building_and_testing.md)
* [Software Contribution Guidelines](doc/guidelines.md)
* [Introductory paper](https://www.spiedigitallibrary.org/proceedings/Download?fullDOI=10.1117%2F12.2529237) (open-access)
* [XL Overview](doc/xl_overview.md) - a brief introduction to the
source code modules
* [JPEG XL committee draft](https://arxiv.org/abs/1908.03565)
* [Introductory paper](https://www.spiedigitallibrary.org/proceedings/Download?fullDOI=10.1117%2F12.2529237)
* JPEG XL white paper with overview of applications and coding tools:
WG1 output document number wg1n86059
### Development process
* [Docker setup - **start here**](doc/developing_in_docker.md)
* [Building on Debian](doc/developing_in_debian.md) - for experts only
* [More information on testing/build options](doc/building_and_testing.md)
* [Git guide for JPEG XL](doc/developing_in_gitlab.md) - for developers only
......@@ -36,18 +36,21 @@ fi
# Whether we should post a message in the MR when the build fails.
POST_MESSAGE_ON_ERROR="${POST_MESSAGE_ON_ERROR:-1}"
# Version inferred from the CI variables.
CI_COMMIT_SHA=${CI_COMMIT_SHA:-$(git log | head -n 1 | cut -b 8-)}
JPEGXL_VERSION=${JPEGXL_VERSION:-${CI_COMMIT_SHA:0:8}}
echo "Version: $JPEGXL_VERSION"
# Convenience flag to pass both CMAKE_C_FLAGS and CMAKE_CXX_FLAGS
CMAKE_FLAGS=${CMAKE_FLAGS:-}
CMAKE_C_FLAGS=${CMAKE_C_FLAGS:-${CMAKE_FLAGS}}
CMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS:-${CMAKE_FLAGS}}
CMAKE_FLAGS="${CMAKE_FLAGS:-} -DJPEGXL_VERSION=\\\"${JPEGXL_VERSION}\\\""
CMAKE_C_FLAGS="${CMAKE_C_FLAGS:-} ${CMAKE_FLAGS}"
CMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS:-} ${CMAKE_FLAGS}"
CMAKE_CROSSCOMPILING_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR:-}
CMAKE_EXE_LINKER_FLAGS=${CMAKE_EXE_LINKER_FLAGS:-}
CMAKE_MODULE_LINKER_FLAGS=${CMAKE_MODULE_LINKER_FLAGS:-}
CMAKE_SHARED_LINKER_FLAGS=${CMAKE_SHARED_LINKER_FLAGS:-}
CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE:-}
# Version inferred from the CI variables.
CI_COMMIT_SHA=${CI_COMMIT_SHA:-}
JPEGXL_VERSION=${JPEGXL_VERSION:-${CI_COMMIT_SHA:0:8}}
# Benchmark parameters
STORE_IMAGES=${STORE_IMAGES:-1}
......@@ -251,7 +254,6 @@ cmake_configure() {
-DCMAKE_MODULE_LINKER_FLAGS="${CMAKE_MODULE_LINKER_FLAGS}"
-DCMAKE_SHARED_LINKER_FLAGS="${CMAKE_SHARED_LINKER_FLAGS}"
-DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}"
-DJPEGXL_VERSION="${JPEGXL_VERSION}"
)
if [[ -n "${BUILD_TARGET}" ]]; then
# If set, BUILD_TARGET must be the target triplet such as
......@@ -276,6 +278,11 @@ cmake_configure() {
args+=(-DPKG_CONFIG_EXECUTABLE="${pkg_config}")
fi
fi
if [[ -n "${CMAKE_CROSSCOMPILING_EMULATOR}" ]]; then
args+=(
-DCMAKE_CROSSCOMPILING_EMULATOR="${CMAKE_CROSSCOMPILING_EMULATOR}"
)
fi
cmake "${args[@]}" "[email protected]"
}
......@@ -408,7 +415,7 @@ cmd_asan() {
CMAKE_CXX_FLAGS+=" -DJXL_ENABLE_ASSERT=1 -g -DADDRESS_SANITIZER \
-fsanitize=address ${UBSAN_FLAGS[@]}"
strip_dead_code
cmake_configure "[email protected]"
cmake_configure "[email protected]" -DJPEGXL_ENABLE_TCMALLOC=OFF
export ASAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}"
export UBSAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}"
cmake_build_and_test
......@@ -481,7 +488,8 @@ cmd_msan() {
CMAKE_SHARED_LINKER_FLAGS+=" ${msan_linker_flags[@]}"
strip_dead_code
cmake_configure "[email protected]" \
-DCMAKE_CROSSCOMPILING=1 -DRUN_HAVE_STD_REGEX=0 -DRUN_HAVE_POSIX_REGEX=0
-DCMAKE_CROSSCOMPILING=1 -DRUN_HAVE_STD_REGEX=0 -DRUN_HAVE_POSIX_REGEX=0 \
-DJPEGXL_ENABLE_TCMALLOC=OFF
export MSAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}"
export UBSAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}"
cmake_build_and_test
......@@ -685,17 +693,31 @@ cmd_cpuset() {
mycpuset="/dev/cpuset${mycpuset}"
# Check that the directory exists:
[[ -d "${mycpuset}" ]]
if [[ -e "${mycpuset}/cpuset.cpus" ]]; then
echo "${newset}" >"${mycpuset}/cpuset.cpus"
else
echo "${newset}" >"${mycpuset}/cpus"
fi
}
# Return the encoding/decoding speed from the Stats output.
_speed_from_output() {
local speed="[email protected]"
speed="${speed%% MP/s*}"
speed="${speed##* }"
echo "${speed}"
}
# Run benchmarks on ARM for the big and little CPUs.
cmd_arm_benchmark() {
local benchmarks=(
# Lossy options:
"--adaptive_reconstruction=1 --distance=1.0 --speed=fast"
"--adaptive_reconstruction=0 --distance=1.0 --speed=fast"
"--adaptive_reconstruction=1 --distance=8.0 --speed=fast"
"--adaptive_reconstruction=0 --distance=8.0 --speed=fast"
"--jpeg1 --jpeg_quality 90"
"--jpeg1 --jpeg_quality 85 --jpeg_420"
"--adaptive_reconstruction=1 --distance=1.0 --speed=cheetah"
"--adaptive_reconstruction=0 --distance=1.0 --speed=cheetah"
"--adaptive_reconstruction=1 --distance=8.0 --speed=cheetah"
"--adaptive_reconstruction=0 --distance=8.0 --speed=cheetah"
"--modular-group -Q 90"
"--modular-group -Q 50"
# Lossless options:
......@@ -709,6 +731,11 @@ cmd_arm_benchmark() {
"--modular-group -Q 97"
)
local brunsli_benchmarks=(
"--num_reps=6 --quant=0"
"--num_reps=6 --quant=20"
)
local images=(
"third_party/testdata/imagecompression.info/flower_foveon.png"
)
......@@ -721,48 +748,85 @@ cmd_arm_benchmark() {
"${RUNNER_CPU_LITTLE%%-*}"
"${RUNNER_CPU_BIG%%-*}"
)
local jpg_dirname="third_party/corpora/jpeg"
mkdir -p "${jpg_dirname}"
local jpg_images=()
local jpg_qualities=( 50 80 95 )
for src_img in "${images[@]}"; do
for q in "${jpg_qualities[@]}"; do
local jpeg_name="${jpg_dirname}/"$(basename "${src_img}" .png)"-q${q}.jpg"
"${BUILD_DIR}/tools/cjpegxl" --jpeg1 --jpeg_quality "${q}" \
"${src_img}" "${jpeg_name}"
jpg_images+=("${jpeg_name}")
done
done
local output_dir="${BUILD_DIR}/benchmark_results"
mkdir -p "${output_dir}"
local runs_file="${output_dir}/runs.txt"
if [[ ! -e "${runs_file}" ]]; then
echo -e "flags\tsrc_img\tsrc size\tsrc pixels\tcpuset\tenc size (B)\tdec speed (MP/s)" |
echo -e "flags\tsrc_img\tsrc size\tsrc pixels\tcpuset\tenc size (B)\tenc speed (MP/s)\tdec speed (MP/s)" |
tee -a "${runs_file}"
fi
mkdir -p "${BUILD_DIR}/arm_benchmark"
local flags
local src_img
for src_img in "${images[@]}"; do
for src_img in "${jpg_images[@]}" "${images[@]}"; do
local src_img_hash=$(sha1sum "${src_img}" | cut -f 1 -d ' ')
for flags in "${benchmarks[@]}"; do
cmd_cpuset "${RUNNER_CPU_ALL}"
local img_benchmarks=("${benchmarks[@]}")
local enc_binary="${BUILD_DIR}/tools/cjpegxl"
local src_ext="${src_img##*.}"
if [[ "${src_ext}" == "jpg" ]]; then
img_benchmarks=("${brunsli_benchmarks[@]}")
enc_binary="${BUILD_DIR}/tools/cbrunsli"
fi
for flags in "${img_benchmarks[@]}"; do
# Encoding step.
local enc_file_hash="$flags || ${src_img} || ${src_img_hash}"
enc_file_hash=$(echo "${enc_file_hash}" | sha1sum | cut -f 1 -d ' ')
local enc_file="${BUILD_DIR}/arm_benchmark/${enc_file_hash}.jxl"
for cpu_conf in "${cpu_confs[@]}"; do
cmd_cpuset "${cpu_conf}"
local enc_file="${BUILD_DIR}/${enc_file_hash}.bin"
echo "Encoding with: img=${src_img} cpus=${cpu_conf} enc_flags=${flags}"
local enc_output
if [[ "${flags}" == *"modular-group"* ]]; then
# We don't benchmark encoding speed in this case.
if [[ ! -f "${enc_file}" ]]; then
"${BUILD_DIR}/tools/cjpegxl" ${flags} "${src_img}" "${enc_file}.tmp"
cmd_cpuset "${RUNNER_CPU_ALL}"
"${enc_binary}" ${flags} "${src_img}" "${enc_file}.tmp"
mv "${enc_file}.tmp" "${enc_file}"
cmd_cpuset "${cpu_conf}"
fi
enc_output=" ?? MP/s"
else
wait_for_temp
enc_output=$("${enc_binary}" ${flags} "${src_img}" "${enc_file}.tmp" \
2>&1 | grep -F "MP/s [")
mv "${enc_file}.tmp" "${enc_file}"
fi
local enc_speed=$(_speed_from_output "${enc_output}")
local enc_size=$(stat -c "%s" "${enc_file}")
for cpu_conf in "${cpu_confs[@]}"; do
cmd_cpuset "${cpu_conf}"
echo "Decoding with: img=${src_img} cpus=${cpu_conf} enc_flags=${flags}"
local dec_output
wait_for_temp
dec_output=$("${BUILD_DIR}/tools/djpegxl" "${enc_file}" \
--num_reps=5 2>&1 | grep -F "MP/s")
--num_reps=5 2>&1 | grep -F "MP/s [")
local img_size=$(echo "${dec_output}" | cut -f 1 -d ',')
local img_size_x=$(echo "${img_size}" | cut -f 1 -d ' ')
local img_size_y=$(echo "${img_size}" | cut -f 3 -d ' ')
local img_size_px=$(( ${img_size_x} * ${img_size_y} ))
local dec_speed=$(echo "${dec_output}" | cut -f 5 -d ' ')
local dec_speed=$(_speed_from_output "${dec_output}")
# Record entry in a tab-separated file.
echo -e "${flags}\t${src_img}\t${img_size}\t${img_size_px}\t${cpu_conf}\t${enc_size}\t${dec_speed}" |
local src_img_base=$(basename "${src_img}")
echo -e "${flags}\t${src_img_base}\t${img_size}\t${img_size_px}\t${cpu_conf}\t${enc_size}\t${enc_speed}\t${dec_speed}" |
tee -a "${runs_file}"
done
done
......@@ -935,6 +999,7 @@ These optional environment variables are forwarded to the cmake call as
parameters:
- CMAKE_C_FLAGS
- CMAKE_CXX_FLAGS
- CMAKE_CROSSCOMPILING_EMULATOR
- CMAKE_EXE_LINKER_FLAGS
- CMAKE_MODULE_LINKER_FLAGS
- CMAKE_SHARED_LINKER_FLAGS
......
# Building and Testing
This file describes the building and testing facilities provided by the `ci.sh`
script. It assumes you already have the build environment set up, preferably
Docker (see [instructions](doc/building_in_docker.md)).
## Basic building
If you only want to build the project but not modify it you can run:
To build the JPEG XL software and run its unit tests, run:
```bash
./ci.sh release
```
Running `./ci.sh` with no parameters shows a list of available commands. For
example, you can run `opt` for optimized developer builds with symbols or
`debug` for debug builds which do not have NDEBUG defined and therefore include
more runtime debug information.
## Sending patches
## Testing
Before sending a patch, make sure your patch conforms to the
[project guidelines](guidelines.md) and test it following these steps.
`./ci.sh` build commands including `release`, `opt`, etc. will also run tests.
You can set the environment variable `SKIP_TEST=1` to skip this.
You will need to install a few other dependencies for verifying these patches,
which are used by the `./ci.sh` helper command.
It is possible to manually run the tests, for example:
```bash
sudo apt install clang-format-6.0 clang-tidy-6.0 curl parallel
cd build/ && ctest -j32 --output-on-failure
```
### Build and unittest checks
`-j32` specifies the number of parallel tasks.
Your patch must build in the "release" and "debug" configurations at least, and
pass all the tests. The "opt" configuration will be tested in the Merge Request
pipeline which is equivalent to a "release" build with debug information.
It is also possible for faster iteration to run a specific test binary directly.
googletest binaries support a glob filter to match the name of the test, for
example `TransposeTest*` will execute all the test names that start with
`TransposeTest`.
```bash
./ci.sh opt
build/dct_test --gtest_filter=TransposeTest.Transpose8
```
All of the tests must pass. If you added new functions, you should also add
tests to these functions in the same commit.
If you want to run multiple tests from different binaries, you can pass a
regular expression filter to ctest. Note that this is a regular expression
instead of a glob match like in the `gtest_filter`.
```bash
cd build/ && ctest -j32 --output-on-failure -R '.*Transpose.*'
```
## Other commands
Running `./ci.sh` with no parameters shows a list of available commands. For
example, you can run `opt` for optimized developer builds with symbols or
`debug` for debug builds which do not have NDEBUG defined and therefore include
more runtime debug information.
### Cross-compiling
......@@ -92,34 +103,6 @@ tidy`. Note that this will report all the problems encountered in any file that
was modified by one of your commits, not just on the lines that your commits
modified.
## Testing
All of the `./ci.sh` workflows like `release`, `opt`, etc will run the test
after building. It is possible to manually run the tests, for example:
```bash
cd build/
ctest -j32 --output-on-failure