Commit 379ebee2 authored by Alexander (asac) Sack's avatar Alexander (asac) Sack
Browse files

Merge branch 'develop' into 'master'

Merge develop to master to prepare 027 release

See merge request !330
parents b4685a85 e874bf77
Pipeline #429035290 passed with stage
in 5 minutes and 7 seconds
......@@ -26,7 +26,8 @@ build-tags:
- tempdir=`mktemp -d`
- docker run --rm $CONTAINER_IMAGE:$CI_BUILD_REF | tar -C $tempdir -x
- tar -cvzf pvr-$CI_BUILD_TAG.linux.amd64.tar.gz -C $tempdir/bin/linux_amd64 pvr
- tar -cvzf pvr-$CI_BUILD_TAG.linux.armv6.tar.gz -C $tempdir/bin/linux_armv6 pvr
- tar -cvzf pvr-$CI_BUILD_TAG.linux.arm.tar.gz -C $tempdir/bin/linux_arm pvr
- tar -cvzf pvr-$CI_BUILD_TAG.linux.arm64.tar.gz -C $tempdir/bin/linux_arm64 pvr
- tar -cvzf pvr-$CI_BUILD_TAG.darwin.amd64.tar.gz -C $tempdir/bin/darwin_amd64 pvr
- echo CWD `pwd`
- sh -c "cd $tempdir/bin/windows_386/; zip pvr-$CI_BUILD_TAG.windows.x32.zip pvr.exe"
......@@ -36,7 +37,8 @@ build-tags:
artifacts:
paths:
- pvr-$CI_BUILD_TAG.linux.amd64.tar.gz
- pvr-$CI_BUILD_TAG.linux.armv6.tar.gz
- pvr-$CI_BUILD_TAG.linux.arm64.tar.gz
- pvr-$CI_BUILD_TAG.linux.arm.tar.gz
- pvr-$CI_BUILD_TAG.darwin.amd64.tar.gz
- pvr-$CI_BUILD_TAG.windows.x32.zip
- pvr-$CI_BUILD_TAG.windows.x64.zip
......@@ -54,7 +56,8 @@ build-master:
- tempdir=`mktemp -d`
- docker run --rm $CONTAINER_IMAGE:$CI_BUILD_REF | tar -C $tempdir -x
- tar -cvzf pvr.linux.amd64.tar.gz -C $tempdir/bin/linux_amd64 pvr
- tar -cvzf pvr.linux.armv6.tar.gz -C $tempdir/bin/linux_armv6 pvr
- tar -cvzf pvr.linux.arm.tar.gz -C $tempdir/bin/linux_arm pvr
- tar -cvzf pvr.linux.arm64.tar.gz -C $tempdir/bin/linux_arm64 pvr
- tar -cvzf pvr.darwin.amd64.tar.gz -C $tempdir/bin/darwin_amd64 pvr
- echo CWD `pwd`
- sh -c "cd $tempdir/bin/windows_386/; zip pvr.windows.x32.zip pvr.exe"
......@@ -64,7 +67,8 @@ build-master:
artifacts:
paths:
- pvr.linux.amd64.tar.gz
- pvr.linux.armv6.tar.gz
- pvr.linux.arm.tar.gz
- pvr.linux.arm64.tar.gz
- pvr.darwin.amd64.tar.gz
- pvr.windows.x32.zip
- pvr.windows.x64.zip
......@@ -82,7 +86,8 @@ build:
- tempdir=`mktemp -d`
- docker run --rm $CONTAINER_IMAGE:$CI_BUILD_REF | tar -C $tempdir -x
- tar -cvzf pvr.linux.amd64.tar.gz -C $tempdir/bin/linux_amd64 pvr
- tar -cvzf pvr.linux.armv6.tar.gz -C $tempdir/bin/linux_armv6 pvr
- tar -cvzf pvr.linux.arm.tar.gz -C $tempdir/bin/linux_arm pvr
- tar -cvzf pvr.linux.arm64.tar.gz -C $tempdir/bin/linux_arm64 pvr
- tar -cvzf pvr.darwin.amd64.tar.gz -C $tempdir/bin/darwin_amd64 pvr
- echo CWD `pwd`
- sh -c "cd $tempdir/bin/windows_386/; zip pvr.windows.x32.zip pvr.exe"
......@@ -92,7 +97,8 @@ build:
artifacts:
paths:
- pvr.linux.amd64.tar.gz
- pvr.linux.armv6.tar.gz
- pvr.linux.arm.tar.gz
- pvr.linux.arm64.tar.gz
- pvr.darwin.amd64.tar.gz
- pvr.windows.x32.zip
- pvr.windows.x64.zip
......
......@@ -15,7 +15,7 @@ RUN go get
FROM src as linux_riscv64
RUN apk update; apk add git
RUN CGO_ENABLED=0 GOOS=linux GOARCH=riscv64 go build -o /go/bin/linux_arm64/pvr -v .
RUN CGO_ENABLED=0 GOOS=linux GOARCH=riscv64 go build -o /go/bin/linux_riscv64/pvr -v .
# build amd64 linux static
......@@ -24,13 +24,13 @@ FROM src as linux_amd64
RUN apk update; apk add git
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /go/bin/linux_amd64/pvr -v .
# build armv6 linux static
FROM src as linux_armv6
# build arm linux static
FROM src as linux_arm
RUN apk update; apk add git
RUN CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=5 go build -o /go/bin/linux_armv6/pvr -v .
RUN CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=5 go build -o /go/bin/linux_arm/pvr -v .
# build armv6 linux static
# build arm64 linux static
FROM src as linux_arm64
RUN apk update; apk add git
......@@ -62,7 +62,8 @@ RUN apk update && apk add ca-certificates && rm -rf /var/cache/apk/*
WORKDIR /work
COPY --from=linux_amd64 /go/bin /pkg/bin
COPY --from=linux_armv6 /go/bin /pkg/bin
COPY --from=linux_arm /go/bin /pkg/bin
COPY --from=linux_arm64 /go/bin /pkg/bin
COPY --from=windows_386 /go/bin /pkg/bin
COPY --from=windows_amd64 /go/bin /pkg/bin
COPY --from=darwin_amd64 /go/bin /pkg/bin
......
......@@ -1029,6 +1029,63 @@ Application added
```
## How to use rootfs type
We now can create an app, install an app, and update an app using as source a root filesystem, that filesystem could be as a tar in local for the computer is running the PVR, could be a plain folder with the filesystem inside, or could be an URL where the tar is to be downloaded.
#### Add app from rootfs
```
pvr app add --from=~/Desktop/pvwebstatus -t rootfs pvwebstatus
```
This will create a new application folder with these files:
```
pvwebstatus/
├── lxc.container.conf
├── root.squashfs
├── root.squashfs.rootfs-digest
├── run.json
└── src.json
```
The Source file will have a rootfs_url argument and rootfs_digest to track the filesystem digest.
If you need to add some docker_config on creation to the app, you could use all the parameters supported. Example:
```
pvr app add --from=~/Desktop/pvwebstatus --config-json=pvwebstatus.config.json -t rootfs pvwebstatus
```
Where the `pvwebstatus.config.json` is a JSON configuration.
```json
{
"ArgsEscaped": true,
"Cmd": [
"/app/pantavisor-web-status"
],
"Env": [
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
],
"WorkingDir": "/app"
}
```
## How to use pvr type
Another source that could be used as a source for an application is a device description inside your computer or from a PV repository URL.
#### Add app from pvr
You could use any PVR repository compatible URL, which could be a local .pvr folder or a remote repository URL.
```
pvr app add -t pvr --from https://pvr.pantahub.com/highercomve/one_marketplace_production#tailscale tailscale
```
## pvr app info <APP_NAME>
pvr app info <appname> :output info and state of appname
......@@ -1093,6 +1150,21 @@ Application updated
```
#### Update app from rootfs
Here you can update like a normal docker container:
```
pvr app update pvwebstatus
```
But you can update from another tar or an URL
```
pvr app update --from=~/Desktop/pvwebstatus.tar pvwebstatus
```
## pvr deploy <deploy-dir> [source-repos]+
......@@ -1125,4 +1197,111 @@ Commands currently supported are:
* pvr sig update - updates a committed signature from the _pvs/ hierarchy to be validate against committed state
* pvr sig ls - list files covered by signatures in _sigs/ hieararchy; by default sig ls will show signature info while considering _all_ signatures in the system state
\ No newline at end of file
## PVR sig with CA commands
```
commit c2d6f1450422b89d90948499c7cd6dd6949e5df3 (HEAD -> feature/pvs-ca, origin/feature/pvs-ca)
Author: Alexander Sack <asac@pantacor.com>
Date: Wed Oct 20 17:58:50 2021 +0200
add support for using x509 cert chains using x5c jws header to determine trust in pvr signatures
* introduce new --x5c argument pvr app sig command to provide the chain to include in pvr sig add and update commands
* introduce --cacerts argument to pvr sig commands to allow to post a trust CACERTS file to use to validate in pvr app ls;
using special value _system will use the system cert store to validate ca chain
* pubkey validation now allows to have multiple trusted pubkeys in the file referenced by --pubkey
* document this feature in README.md
Example 1: "add signature with trust ca chain"
Below statement injects the myKey.crt as the trust chain into the jws.
If you have intermediates your .crt file would need to include those
also in reverse order.
```
pvr sig --x5c ../ca/myKey.crt --key ../ca/myKey.key add --part nginx
```
Example 2: "update signatures with trustchain"
Below will refresh the nginx.json signature and attach myKey.crt as
the trust ca cert chain to validate against root certificates
```
pvr sig --x5c ../ca/myKey.crt --key ../ca/myKey.key update _sigs/nginx.json
```
Example 3: "validate signatures with cert pool in file"
Below you can see how to validate signature with ca cert pool in file myCA.pem.
```
pvr sig --cacerts ../ca/myCA.pem ls --part _sigs/nginx.json
```
Example 4: use system ca cert pool to validate signature
For this you have to put your myCA.pem into one of the system folders for
trusted certificates. e.g. /etc/ssl/certs
```
pvr sig ls --part _sigs/nginx.json
```
```
# PVR dm commands (device-mapper)
pvr device mapper support for container volumes allows for an easy way to
postprocess the squashfs volumes produced by pvr app add etc. in a way that
the pantavisor device mapper addon can mount volumes using device-mapper.
For now dm-verity type device mapper entries are supported by pantavisor
client and hence pvr supports that mode first and foremost for now.
## PVR dm-convert
This command allows you to convert any standard squashfs volume into a
device-mapper mounted volume.
For dm-verity this will create a manifest file in <container>/_dm/<volume>.json,
e.g. os/_dm/root.squashfs.json.
To convert a volume simply use the following command:
```
pvr dm-convert os root.squashfs
```
This will convert the root.squashfs volume of the container 'os' into
a device mapper enabled one.
It will create the os/_dm/root.squashfs.json manifest:
```
$ cat os/_dm/root.squashfs.json | jq .
{
"type": "dm-verity"
"data_device": "root.squashfs",
"hash_device": "root.squashfs.hash",
"root_hash": "88298f349288e685ac2474134ef22bf8f77465cde250c9f698780b5b6d942b96",
}
```
... and also the hash_device in `os/root.squashfs.hash`.
Also it will convert the volume reference in run.json to "dm:<volume", e.g.
`dm:rootfs.squashfs`. This syntax will ensure that pantavisor will not try to
mount the squash himself, but rather delegate that to a device-mapper
volume handler.
You can then `pvr commit` this and post it to a pantavisor device-mapper
enabled device
## PVR dm-apply
This command iterates through the whole committed pristine json of the repo
and updates the hash and manifests for all dm-verity manifests.
This is good if you updated a container and wnat to recalculate the hash file.
......@@ -22,6 +22,7 @@ import (
"strings"
"gitlab.com/pantacor/pvr/libpvr"
"gitlab.com/pantacor/pvr/models"
"github.com/urfave/cli"
)
......@@ -101,6 +102,7 @@ func CommandAppAdd() cli.Command {
ConfigFile: c.String("config-json"),
Volumes: c.StringSlice("volume"),
FormatOptions: c.String("format-options"),
SourceType: c.String("type"),
TemplateArgs: templateArgs,
}
......@@ -108,17 +110,12 @@ func CommandAppAdd() cli.Command {
app.TemplateArgs["PV_RUNLEVEL"] = c.String("runlevel")
}
err = pvr.FindDockerImage(&app)
if err != nil {
return cli.NewExitError(err, 3)
}
err = pvr.AddApplication(app)
if err != nil {
return cli.NewExitError(err, 3)
}
fmt.Println("Application added")
return nil
},
}
......@@ -150,6 +147,12 @@ func CommandAppAdd() cli.Command {
EnvVar: "PVR_SOURCE",
Value: "remote,local",
},
cli.StringFlag{
Name: "type, t",
Usage: fmt.Sprintf("Type of source. available types [%s, %s, %s]", models.SourceTypeDocker, models.SourceTypePvr, models.SourceTypeRootFs),
EnvVar: "PVR_SOURCE_TYPE",
Value: models.SourceTypeDocker,
},
cli.StringFlag{
Name: "runlevel",
Usage: RunlevelFlagUsage,
......
......@@ -21,6 +21,7 @@ import (
"strings"
"gitlab.com/pantacor/pvr/libpvr"
"gitlab.com/pantacor/pvr/models"
"github.com/urfave/cli"
)
......@@ -101,15 +102,12 @@ func CommandAppInstall() cli.Command {
Source: source,
Username: username,
Password: password,
SourceType: c.String("type"),
TemplateArgs: map[string]interface{}{},
}
err = pvr.FindDockerImage(&app)
if err != nil {
fmt.Println("\nSeems like you have an invalid docker digest value in your " + appname + "/src.json file\n")
fmt.Println("\nPlease run \"pvr app update " + appname + " --source=" + c.String("source") + "\" to auto fix it or update docker_digest field by editing " + appname + "/src.json to fix it manually\n")
return cli.NewExitError(err, 3)
}
pvr.SetSourceTypeFromManifest(&app, nil)
err = pvr.InstallApplication(app)
if err != nil {
return cli.NewExitError(err, 3)
......@@ -132,6 +130,11 @@ func CommandAppInstall() cli.Command {
Usage: "Use `PVR_REGISTRY_PASSWORD` for authorization with docker registrar",
EnvVar: "PVR_REGISTRY_PASSWORD",
},
cli.StringFlag{
Name: "type, t",
Usage: fmt.Sprintf("Type of source. available types [%s, %s, %s]", models.SourceTypeDocker, models.SourceTypePvr, models.SourceTypeRootFs),
EnvVar: "PVR_SOURCE_TYPE",
},
cli.StringFlag{
Name: "source",
Usage: SourceFlagUsage,
......
......@@ -21,6 +21,7 @@ import (
"strings"
"gitlab.com/pantacor/pvr/libpvr"
"gitlab.com/pantacor/pvr/models"
"github.com/urfave/cli"
)
......@@ -84,9 +85,12 @@ func CommandAppUpdate() cli.Command {
Platform: c.String("platform"),
Username: c.String("username"),
Password: c.String("password"),
SourceType: c.String("type"),
TemplateArgs: map[string]interface{}{},
}
pvr.SetSourceTypeFromManifest(&app, nil)
if c.String("runlevel") != "" {
app.TemplateArgs["PV_RUNLEVEL"] = c.String("runlevel")
}
......@@ -112,6 +116,11 @@ func CommandAppUpdate() cli.Command {
Usage: "Use `PVR_REGISTRY_PASSWORD` for authorization with docker registrar",
EnvVar: "PVR_REGISTRY_PASSWORD",
},
cli.StringFlag{
Name: "type, t",
Usage: fmt.Sprintf("Type of source. available types [%s, %s, %s]", models.SourceTypeDocker, models.SourceTypePvr, models.SourceTypeRootFs),
EnvVar: "PVR_SOURCE_TYPE",
},
cli.StringFlag{
Name: "source",
Usage: SourceFlagUsage,
......
//
// Copyright 2021 Pantacor Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package main
import (
"os"
"github.com/urfave/cli"
"gitlab.com/pantacor/pvr/libpvr"
)
func CommandDmApply() cli.Command {
return cli.Command{
Name: "dm-apply",
Aliases: []string{"dma"},
ArgsUsage: "[<component-prefix>]",
Usage: "apply device mapper state transform of a normalized pantavisor state; only apply for _dm matches for platforms starting with '<component-prefix>'",
Action: func(c *cli.Context) error {
wd, err := os.Getwd()
if err != nil {
return err
}
session, err := libpvr.NewSession(c.App)
if err != nil {
return cli.NewExitError(err, 4)
}
pvr, err := libpvr.NewPvr(session, wd)
if err != nil {
return cli.NewExitError(err, 2)
}
if len(c.Args()) > 0 {
err = pvr.DmCVerityApply(c.Args()[0])
} else {
err = pvr.DmCVerityApply("")
}
if err != nil {
return cli.NewExitError(err, 3)
}
return nil
},
Flags: []cli.Flag{},
}
}
//
// Copyright 2021 Pantacor Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
package main
import (
"errors"
"os"
"strings"
"github.com/urfave/cli"
"gitlab.com/pantacor/pvr/libpvr"
)
func CommandDmConvert() cli.Command {
return cli.Command{
Name: "dm-convert",
Aliases: []string{"dmc"},
ArgsUsage: "<container[/volume]> <volume>",
Usage: "convert a volume to device mappers dm-verity based volume",
Action: func(c *cli.Context) error {
wd, err := os.Getwd()
if err != nil {
return err
}
session, err := libpvr.NewSession(c.App)
if err != nil {
return cli.NewExitError(err, 4)
}
pvr, err := libpvr.NewPvr(session, wd)
if err != nil {
return cli.NewExitError(err, 2)
}
if len(c.Args()) < 1 {
return cli.NewExitError(errors.New("missing arguments, see --help"), 2)
}
container := c.Args()[0]
var volume string
if len(c.Args()) == 2 {
volume = c.Args()[1]
} else {
_arr := strings.SplitN(container, "/", 2)
if len(_arr) <= 1 {
err = errors.New("ERROR: volume could not be found; see --help")
return cli.NewExitError(err, 3)
}
container = _arr[0]
volume = _arr[1]
}
err = pvr.DmCVerityConvert(container, volume)
if err != nil {
return cli.NewExitError(err, 4)
}
return nil
},
Flags: []cli.Flag{},
}
}
......@@ -36,10 +36,21 @@ func CommandSig() cli.Command {
EnvVar: "PVR_SIG_KEY",
Usage: "private key in PEM format to use for signing",
},
cli.StringFlag{
Name: "x5c, x",
EnvVar: "PVR_X5C_PATH",
Usage: "path to cert chain to include in jws x5c header. Note: we will not validate that the actual signature can be validated with this one.",
},
cli.StringFlag{
Name: "pubkey, p",
EnvVar: "PVR_SIG_PUBKEY",
Usage: "pubkey in PEM format to use for signing",
Usage: "use specific pubkey store to validate signatures.",
},
cli.StringFlag{
Name: "cacerts, c",
EnvVar: "PVR_SIG_CACERTS",
Usage: "initialize cert pool from file or directory provided in this argument. use __system__ to use system cert store",
Value: "_system_",
},
},
}
......
......@@ -110,6 +110,8 @@ func CommandSigAdd() cli.Command {
return cli.NewExitError("needs a --key argument; see --help.", 126)
}
ops.X5cPath = c.Parent().String("x5c")
err = pvr.JwsSign(name, keyPath, &match, &ops)
if err != nil {
......
......@@ -108,19 +108,16 @@ func CommandSigLs() cli.Command {
}
pubkey := c.Parent().String("pubkey")
if pubkey != "" {
keyFs, err := os.Stat(pubkey)
if pubkey == "" {
return cli.NewExitError(errors.New("ERROR: no signing key provided; see --help"), 10)
}
keyFs, err := os.Stat(pubkey)
if err != nil {
return cli.NewExitError(("ERROR: errors accessing signing key; see --help: " + err.Error()), 11)
}
if err != nil {
return cli.NewExitError(("ERROR: errors accessing signing key; see --help: " + err.Error()), 11)
}
if keyFs.IsDir() {
return cli.NewExitError(("ERROR: signing key is not a file; see --help: " + err.Error()), 12)
if keyFs.IsDir() {
return cli.NewExitError(("ERROR: signing key is not a file; see --help: " + err.Error()), 12)
}
}
_args := c.Args()
......@@ -156,8 +153,10 @@ func CommandSigLs() cli.Command {
var resultSummary libpvr.JwsVerifySummary
var verifySummary []libpvr.JwsVerifySummary
cacerts := c.Parent().String("cacerts")
for _, v := range args {
w, err := pvr.JwsVerifyPvs(pubkey, v)
w, err := pvr.JwsVerifyPvs(pubkey, cacerts, v)
if errors.Is(err, os.ErrNotExist) {
return cli.NewExitError("ERROR: signature file does not exist with name "+v, 125)
}
......
......@@ -65,6 +65,7 @@ func CommandSigUpdate() cli.Command {
}
ops := libpvr.PvsOptions{}
ops.X5cPath = c.Parent().String("x5c")
keyPath := c.Parent().String("key")
if keyPath == "" {
......