Commit a84f2a30 authored by Jason Plum's avatar Jason Plum 📈 Committed by Marin Jankovski

Registry: using upstream container

parent 32e8f21f
config.yaml
config.yml
apiVersion: v1
name: registry
version: 0.0.1
description: "Helm chart for Container Registry"
description: "Helm chart for Container Registry with GitLab"
keywords:
- registry
- container
- docker
- gitlab
home: https://github.com/docker/distribution
sources:
- https://github.com/docker/distribution
......
{{- if .Values.enabled -}}
{{- $httpSecret := randAlphaNum 128 | b64enc -}}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ template "fullname" . }}
labels:
app: {{ template "fullname" . }}
data:
config.yml: |
version: 0.1
log:
level: warn
fields:
service: registry
# health:
# storagedriver:
# enabled: true
# interval: 10s
# threshold: 3
http:
debug:
addr: :5001
headers:
X-Content-Type-Options: [nosniff]
# Filled from chart
http:
addr: :{{ .Values.service.internalPort }}
secret: {{ default $httpSecret .Values.registry.httpSecret }}
auth:
token:
realm: {{ .Values.registry.authEndpoint }}/jwt/auth
service: {{ .Values.registry.tokenService }}
issuer: {{ .Values.registry.tokenIssuer | quote }}
rootcertbundle: /etc/registry-secret/{{ .Values.registry.certBundle.bundleName }}
storage:
{{- if .Values.registry.storage }}
{{ toYaml .Values.registry.storage | indent 6}}
{{- end }}
{{- end -}}
{{- if .Values.enabled -}}
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
......@@ -8,39 +9,34 @@ metadata:
release: "{{ .Release.Name }}"
heritage: "{{ .Release.Service }}"
spec:
replicas: 1
replicas: {{ .Values.registry.replicas }}
template:
metadata:
labels:
app: {{ template "fullname" . }}
spec:
containers:
- name: registry
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
imagePullPolicy: {{ default "" .Values.image.pullPolicy | quote }}
env:
- name: REGISTRY_HTTP_ADDR
valueFrom:
configMapKeyRef:
name: {{ template "fullname" . }}
key: http.addr
- name: REGISTRY_HTTP_SECRET
valueFrom:
configMapKeyRef:
name: {{ template "fullname" . }}
key: http.secret
- name: REGISTRY_AUTH_TOKEN_REALM
valueFrom:
configMapKeyRef:
name: {{ template "fullname" . }}
key: auth.token.realm
- name: REGISTRY_AUTH_TOKEN_SERVICE
valueFrom:
configMapKeyRef:
name: {{ template "fullname" . }}
key: auth.token.service
- name: REGISTRY_AUTH_TOKEN_ISSUER
valueFrom:
configMapKeyRef:
name: {{ template "fullname" . }}
key: auth.token.issuer
- name: registry
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
imagePullPolicy: {{ default "" .Values.image.pullPolicy | quote }}
volumeMounts:
- name: {{ .Release.Name }}-secret
mountPath: '/etc/registry-secret'
readOnly: true
- name: config-volume
mountPath: /etc/docker/registry/
readOnly: true
resources:
{{ toYaml .Values.resources | indent 12 }}
volumes:
- name: {{ .Release.Name }}-secret
secret:
secretName: {{ .Values.registry.certBundle.secretName }}
items:
- key: {{ .Values.registry.certBundle.bundleName }}
path: {{ .Values.registry.certBundle.bundleName }}
defaultMode: 0400
- name: config-volume
configMap:
name: {{ template "fullname" . }}
{{- end -}}
{{- if and .Values.ingress.enabled .Values.enabled -}}
{{- $serviceName := include "fullname" . -}}
{{- $servicePort := .Values.service.externalPort -}}
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: {{ template "fullname" . }}
labels:
app: {{ template "fullname" . }}
chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
release: "{{ .Release.Name }}"
heritage: "{{ .Release.Service }}"
annotations:
{{- range $key, $value := .Values.ingress.annotations }}
{{ $key }}: {{ $value | quote }}
{{- end }}
spec:
rules:
{{- range $host := .Values.ingress.hosts }}
- host: {{ $host.name }}
http:
paths:
- path: /
backend:
serviceName: {{ $serviceName }}
servicePort: {{ $servicePort }}
{{- end -}}
{{- if .Values.ingress.tls }}
tls:
{{ toYaml .Values.ingress.tls | indent 4 }}
{{- end -}}
{{- end -}}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ template "fullname" . }}
labels:
app: {{ template "fullname" . }}
data:
http.addr: {{ .Values.service.clusterIP }}:{{ .Values.service.internalPort }}
http.secret: {{ .Values.registry.httpSecretName | quote }}
auth.token.realm: {{ .Values.registry.authEndpoint }}/jwt/auth
auth.token.service: {{ .Values.registry.httpSecretName | quote }}
auth.token.issuer: {{ .Values.registry.httpSecretName | quote }}
{{- if .Values.enabled -}}
apiVersion: v1
kind: Service
metadata:
name: {{ template "fullname" . }}
labels:
chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
app: {{ template "fullname" . }}
spec:
type: {{ .Values.service.type }}
ports:
......@@ -13,3 +15,4 @@ spec:
name: {{ .Values.service.name }}
selector:
app: {{ template "fullname" . }}
{{- end -}}
image:
repository: registry
tag: '2.6.1'
tag: '2.6'
pullPolicy: 'IfNotPresent'
service:
name: registry
......@@ -8,10 +8,46 @@ service:
externalPort: 5000
internalPort: 5000
clusterIP: '0.0.0.0'
# define some sane resource requests and limitations
resources: {}
# limits:
# cpu: 200m
# memory: 1024Mi
# requests:
# cpu: 100m
# memory: 128Mi
# Chart operation controls
enabled: true
registry:
httpSecretName: 'secret'
httpSecretKeyName: 'secret-key'
# Will be auto-populated if not present (`{{ randAlphaNum 128 | b64enc | quote }}`)
httpSecret: ''
storage: {}
authEndpoint: 'https://gitlab.example.com'
# Protocol/Host only. '/jwt/auth' will be appended.
authEndpoint: 'https://gitlab.example.local'
# Do not change this
tokenService: 'container_registry'
# This must match your GitLab Rails configuration
tokenIssuer: 'gitlab-registry-issuer'
# Public side of the certificate bundle used to sign JWT auth requests
certBundle:
secretName: 'gitlab-registry-certbundle'
bundleName: 'gitlab-registry.crt'
# if replicas is >1, shared storage MUST be used.
replicas: 1
# Chart specific Ingress
ingress:
enabled: false
# Used to create Ingress record (should used with service.type: ClusterIP).
hosts:
# - name: registry.example.local
annotations:
kubernetes.io/ingress.class: nginx
# kubernetes.io/tls-acme: "true"
tls:
# Secrets must be manually created in the namespace.
# - hosts:
# - registry.example.local
# secretName: registry-example-tls
......@@ -21,9 +21,30 @@ Check that `kubectl` will be tagetting your Minikube install.
- List available clusters for `kubectl`: `kubectl config get-clusters`
- Change to `minikube`: `kubectl config set-cluster minikube`
Initialize Helm, and install the Tiller service with `helm init`. If your cluster already had Tiller, run `helm init --upgrade` to ensure that the deployed version of Tiller matches.
Initialize Helm, and install the Tiller service with `helm init`. If your cluster
already had Tiller, run `helm init --upgrade` to ensure that the deployed version of Tiller matches.
## Useful documentation for Helm
The Build Team has a [training presentation for Helm Charts](https://docs.google.com/presentation/d/1CStgh5lbS-xOdKdi3P8N9twaw7ClkvyqFN3oZrM1SNw/present)
### Templates
Templating in Helm is done via golang's [text/template][] and [sprig][].
Some information on how all the inner workings behave:
- [Functions and Pipelines][helm-func-pipeline]
- [Subcharts and Globals][helm-subchart-global]
### Tips and Tricks
https://github.com/kubernetes/helm/blob/master/docs/charts_tips_and_tricks.md
[helm]: https://helm.sh
[helm-using]: https://docs.helm.sh/using_helm
[k8s-io]: https://kubernetes.io/
[text/template]: https://golang.org/pkg/text/template/
[sprig]: https://godoc.org/github.com/Masterminds/sprig
[helm-func-pipeline]: https://github.com/kubernetes/helm/blob/master/docs/chart_template_guide/functions_and_pipelines.md
[helm-subchart-global]: https://github.com/kubernetes/helm/blob/master/docs/chart_template_guide/subcharts_and_globals.md
......@@ -83,6 +83,13 @@ Minikube supports [PersistenVolumes](k8s-pv) of the `hostPath` type, which are m
Further details, and listings of directories that _do_ persist, can be found [in the getting started guide](https://kubernetes.io/docs/getting-started-guides/minikube/#persistent-volumes)
### Enable Addons
Minikube handles some features apart from the base configuration. For the development of this project, we'll need access to `Ingress` and `kube-dns` will be helpful.
- `minikube addons enable ingress`
- `minikube addons enable kube-dns`
### Connecting to the dashboard
You can find the URL for the dashboard by calling `minikube dashboard --url`.
......
# Using the Container Registry
The `registry` sub-chart provides the Registry component to a complete cloud-native
GitLab deployment on Kubernetes. This sub-chart makes use of the upstream [registry][]
[container][docker-distribution-library] containing [Docker Distribution][docker-distribution]. This chart is composed of 3 primary parts: [Service][], [Deployment][], and [ConfigMap][]. An additional optional [Ingress][] has been
provided to allow separation from the global [Ingress](../README.md#ingress) as provided by the parent chart.
All configuration is handled according to the official [Registry configuration documentation][docker-distribution-config-docs]
using `/etc/docker/registry/config.yml` variables provided to the [Deployment][], populated from the [ConfigMap][]. The [ConfigMap][] overrides the upstream defaults, but is [based upon them][registry-config].
## Design Choices
A Kubernetes `Deployment` was chosen as the deployment method for this chart to
allow for simple scaling of instances, while allowing for [rolling-update](https://kubernetes.io/docs/user-guide/kubectl/v1.7/#rolling-update)s.
This chart makes use of only two secrets:
- `certBundle`: A secret that will contain the public certificate bundle to verify
the authentication tokens provided by the associated GitLab instance(s). See
[documentation](https://docs.gitlab.com/ee/administration/container_registry.html#disable-container-registry-but-use-gitlab-as-an-auth-endpoint) on using GitLab as an auth endpoint.
- *optional*: The secret which will contain the SSL certificates for the HTTPS
termination by the [Ingress][]. This secret follows the requirements set forth in
[Kubernetes Ingress's TLS section][kubernetes-ingress]. If you chose to use
the global [Ingress](../README.md#ingress) from the parent chart, this will not
be required at all.
# Configuration
We will describe all the major sections of the configuration below. When configuring from the parent chart, these values will be as such:
```
registry:
enabled:
image:
service:
registry:
```
If you should chose to deploy this chart as a standalone, remove the top level `registry`.
## Enable the sub-chart
They way we've chosen to implement compartmentalized sub-charts includes the ability to disable the components that you may not want in a given deployment. For this reason, the first settings you should decided upon is `enabled:`.
By default, Registry is enabled out of the box. Should you wish to disable it,
set `enabled: false`.
## Configuring the `image`
This section dictates the settings for the container image used by this sub-chart's [Deployment][]. You can change the included version of the Registry and `pullPolicy`.
Default settings:
- `tag: '2.6'`
- `pullPolicy: 'IfNotPresent'`
## Configuring the `service`
This section controls the name and type of the [Service][]. These settings will
be populated by the [values.yml][].
By default, the [Service][] is configured as:
- `type: ClusterIP` on `0.0.0.0`, restricting access to the interal network of the Kubernetes cluster.
- `name:` is set to `registry`.
## Configuring the Registry
The `registry` section of this chart pertains to the configuration of the underlying
[registry][] container. Only most critical values for integration with GitLab are
exposed. For this integration, we make use of the `auth.token.x` settings of
[Docker Distribution][docker-distribution], controlling authentication to the registry via JWT
[authentication tokens](https://docs.docker.com/registry/spec/auth/token/).
#### httpSecret
Field `httpSecret` is a string that correlates to the `http.secret` value of [registry][].
This value will be automatically populated with a random string of 128 alpha-numeric
characters encoded to base64.
You should only need to supply this value when using a load balancer across
multiple clusters. See the following note from the [Registry configuration documents][docker-distribution-config-docs]:
> If you are building a cluster of registries behind a load balancer, you MUST ensure the secret is the same for all registries.
#### authEndpoint
Field `authEndpoint` is a string, providing the URL to the GitLab instance(s) that the [registry][] will authenticate to.
The value should include the protocol and hostname only. The chart template will automatically append the necessary request path. The resulting value will be populated to `auth.token.realm` inside the container.
Example: `authEndpoint: "https://gitlab.example.local"`
#### certBundle
Field `certBundle` is a map containing two items: `secretName` and `bundleName`.
`secretName` is a string containing the name of the [Kubernetes Secret][kubernetes-secret] that houses the certificate bundle to be used to verify the tokens created by the GitLab instance(s).
`bundleName` is the name of the `key` in the `Secret` which houses the certificate
bundle that will be provided to the [registry][] container as `auth.token.rootcertbundle`.
Default Example:
```
certBundle:
secretName: gitlab-registry-certbundle
bundleName: gitlab-registry.crt
```
#### replicas
Field `replicas` is an integer, controlling the number of [registry][] instances to create as a part of the set. This defaults to `1`.
#### storage
Field `storage` is a map, the value of which is taken directly from [Registry Configuration: `storage`](https://docs.docker.com/registry/configuration/#storage). Please refer to that documentation for extended details.
If you chose to use the `filesystem` driver, you will need to provide persistent volumes for this data.
For the sake of resiliency and simplicity, it is recommended to make use of an
external service other than the `filesystem` driver, such as `s3`, `gcs` or `azure`.
## Configuring the Ingress (optional)
This section describes configuring the *optional* dedicated [Ingress][]. By default this is disabled, so you'll have to enable it to make use of the following series of settings. Primarily, these settings will be familiar with [Kubernetes Ingress][kubernetes-ingress] documentation, but slightly simplified thanks to [Helm][helm].
#### enabled
Field `enable:`, boolean
This enables or disables this dedicated [Ingress][].
Default `false`, set `true` to enable.
#### hosts
Field `hosts:`, a list of items in the form of `name: fqdn`
This controls the hostnames accepted by the [Ingress][]. Note that we do not make
use of any other component fields that could be used when defining an `host:`, as
we're only linking to the [Service][] contained in the chart.
#### tls
Field `tls:`, a map of items, per the [Kubernetes Ingress][kubernetes-ingress] documentation.
As the official documentation shows, this field should contain a map including a
list of `hosts` by hostname, and a `secretName` which names the [Kubernetes Secret][kubernetes-secret]
that houses the TLS certificate and key to be used for that hostname. And exmaple
is found below appear as such:
```
tls:
- hosts:
- registry.example.local
secretName: registry-example-tls
```
*Note:* While you may be able to combine `tls` with ACME, it is not tested.
#### annotations
This field is an exact match to the standard `annotations` for [Kubernetes Ingress][kubernetes-ingress].
The default value includes setting of `kubernetes.io/ingress.class: nginx`.
If you need to replace this value, or add additional, you may do so.
When using `nginx` as the `ingress.class`, it is also recommended to add
`ingress.kubernetes.io/proxy-body-size: 1024M`. This controls `client_max_body_size`,
and should be sized appropriately when used in conjunction with the [registry][]
to allow large images.
One example of an additional `annotation` is `kubernetes.io/tls-acme: "true"`
to enable automatic Lets Encrypt as a part of the [Ingress][] in combination with `kube-lego`.
Further details of available `ingress.class: nginx` options can be found
[in the documentation][kubernetes-ingress-nginx-configuration] for the NGINX Ingress.
[registry]: https://hub.docker.com/_/registry/
[docker-distribution]: https://github.com/docker/distribution
[docker-distribution-library]: https://github.com/docker/distribution-library-image
[docker-distribution-config-docs]: https://docs.docker.com/registry/configuration
[registry-config]: https://github.com/docker/distribution-library-image/blob/master/registry/config-example.yml
[Service]: ../../charts/registry/templates/service.yaml
[Deployment]: ../../charts/registry/templates/deployment.yaml
[ConfigMap]: ../../charts/registry/templates/registry-configmap.yaml
[Ingress]: ../../charts/registry/templates/ingress.yaml
[values.yml]: ../../charts/registry/values.yml
[kubernetes-ingress]: https://kubernetes.io/docs/concepts/services-networking/ingress/#tls
[kubernetes-secret]: https://kubernetes.io/docs/concepts/configuration/secret/
[helm]: https://helm.sh
[kubernetes-ingress-nginx-configuration]: https://github.com/kubernetes/ingress/blob/master/controllers/nginx/configuration.md
1. Get the application URL by running these commands:
{{- if .Values.ingress.hostname }}
http://{{- .Values.ingress.hostname }}
{{- else if contains "NodePort" .Values.service.type }}
export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "fullname" . }})
export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
echo http://$NODE_IP:$NODE_PORT
{{- else if contains "LoadBalancer" .Values.service.type }}
NOTE: It may take a few minutes for the LoadBalancer IP to be available.
You can watch the status of by running 'kubectl get svc -w {{ template "fullname" . }}'
export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
echo http://$SERVICE_IP:{{ .Values.service.externalPort }}
{{- else if contains "ClusterIP" .Values.service.type }}
export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app={{ template "fullname" . }}" -o jsonpath="{.items[0].metadata.name}")
echo "Visit http://127.0.0.1:8080 to use your application"
kubectl port-forward $POD_NAME 8080:{{ .Values.service.externalPort }}
{{- else }}
kubectl get ingress
{{- end }}
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: {{ template "fullname" . }}
labels:
chart: "{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}"
spec:
replicas: {{ .Values.replicaCount }}
template:
metadata:
labels:
app: {{ template "fullname" . }}
spec:
containers:
- name: {{ .Chart.Name }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
ports:
- containerPort: {{ .Values.service.internalPort }}
livenessProbe:
httpGet:
path: /
port: {{ .Values.service.internalPort }}
readinessProbe:
httpGet:
path: /
port: {{ .Values.service.internalPort }}
resources:
{{ toYaml .Values.resources | indent 10 }}
{{- if .Values.ingress.enabled -}}
{{- $serviceName := include "fullname" . -}}
{{- $servicePort := .Values.service.externalPort -}}
{{- $releaseName := .Release.Name -}}
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
......@@ -17,13 +17,13 @@ metadata:
spec:
rules:
{{- range $host := .Values.ingress.hosts }}
- host: {{ $host }}
- host: {{ $host.name }}
http:
paths:
- path: /
backend:
serviceName: {{ $serviceName }}
servicePort: {{ $servicePort }}
serviceName: "{{ $releaseName }}-{{ $host.serviceName }}"
servicePort: {{ $host.servicePort }}
{{- end -}}
{{- if .Values.ingress.tls }}
tls:
......
# Default values for gitlab-chart.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
replicaCount: 1
image:
repository: nginx
tag: stable
pullPolicy: IfNotPresent
service:
name: nginx
type: ClusterIP
externalPort: 80
internalPort: 80
ingress:
enabled: false
# Used to create Ingress record (should used with service.type: ClusterIP).
hosts:
- chart-example.local
# - name: registry.example.local
# serviceName: registry
# servicePort: 5000
annotations:
# kubernetes.io/ingress.class: nginx
kubernetes.io/ingress.class: nginx
# kubernetes.io/tls-acme: "true"
tls:
# Secrets must be manually created in the namespace.
# - secretName: chart-example-tls
# hosts:
# - chart-example.local
resources:
limits:
cpu: 100m
memory: 128Mi
requests:
cpu: 100m
memory: 128Mi
# - registry.example.local
registry:
enabled: false
ingress:
enabled: false
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment