Koios API inconsistent with official sources
The Koios API, as provided by koios.rest, is served under /api/v0 endpoints. This is facilitated by an haproxy configuration which separates endpoints that are direct queries from the RPC endpoints. This means it serves things like /rpc/tip under /api/v0/tip while also serving /blocks under /api/v0/blocks for the API.
I added the following configmap item:
haproxy-fetch-grest-poll.sh: |
function init {
export SKIP_UPDATE='Y'
export CURL_TIMEOUT=20
}
init
curl -LO https://raw.githubusercontent.com/cardano-community/guild-operators/${KOIOS_BRANCH}/scripts/grest-helper-scripts/setup-grest.sh
sed -i '/.*# Execution.*/,$d' setup-grest.sh
source setup-grest.sh
function setup_cron_jobs {
return 0
}
mkdir -p ${CNODE_HOME}/files ${CNODE_HOME}/scripts
common_update
cd ${CNODE_HOME}/scripts
curl -LO https://raw.githubusercontent.com/cardano-community/guild-operators/${KOIOS_BRANCH}/scripts/grest-helper-scripts/grest-poll.sh
sed -i 's|#API_COMPARE="http://127.0.0.1:8050"|API_COMPARE="http://koios-api:3000" |' grest-poll.sh
echo "function checkUpdate() { :; };" > env
echo '#!/usr/bin/env bash' > ubuntu-wrapper.sh
echo "if [[ ! -x /usr/bin/jq ]] || [[ ! -x /usr/bin/shasum ]]; then apt-get update && apt-get install -y jq perl; fi" >> ubuntu-wrapper.sh
echo 'exec /grest/scripts/grest-poll.sh ${@}' >> ubuntu-wrapper.sh
chmod +x grest-poll.sh ubuntu-wrapper.sh
I am using the Ubuntu image for haproxy to have bash available, since it's necessary for grest-poll.sh functionality.
Here's the values.yaml I am using with the haproxytech/haproxy Helm chart via Helmfile.
config: |
global
daemon
nbthread 4
maxconn 256
ulimit-n 65536
cpu-map 1/all 1-2
insecure-fork-wanted
external-check
log stdout format raw local0
defaults
mode http
log global
option dontlognull
option http-ignore-probes
option http-server-close
option forwardfor
log-format "%ci:%cp a:%f/%b/%s t:%Tq/%Tt %{+Q}r %ST b:%B C:%ac,%fc,%bc,%sc Q:%sq/%bq"
option dontlog-normal
timeout client 30s
timeout server 30s
timeout connect 3s
timeout server-fin 2s
timeout http-request 5s
frontend app
bind *:80
http-request set-log-level silent
acl is_wss hdr(Upgrade) -i websocket
http-request use-service prometheus-exporter if { path /metrics }
http-request track-sc0 src table flood_lmt_rate
http-request deny deny_status 429 if { sc_http_req_rate(0) gt 250 }
use_backend ogmios_core if { path_beg /api/v0/ogmios } || { path_beg /dashboard.js } || { path_beg /assets } || { path_beg /health } || is_wss
use_backend submitapi if { path_beg /api/v0/submittx }
default_backend grest_core
backend grest_core
balance first
option external-check
acl chktip path -m beg /rpc/tip
acl grestrpcs path_beg -f /grest/files/grestrpcs
http-request set-path "%[path,regsub(^/api/v0/,/)]" if ! grestrpcs
http-request set-path "%[path,regsub(^/,/rpc/)]" if grestrpcs
http-request cache-use grestcache
http-request set-log-level silent if chktip
external-check path "/usr/bin:/bin:/tmp:/sbin:/usr/sbin:/grest/scripts"
external-check command /grest/scripts/ubuntu-wrapper.sh
server local koios-api:3000 check inter 20000
{{- if eq .Values.cardano_network "mainnet" }}
server koios-ssl api.koios.rest:443 backup ssl verify none
{{- else }}
server koios-ssl testnet.koios.rest:443 backup ssl verify none
{{- end }}
http-response cache-store grestcache
http-response set-header X-Frame-Options: DENY
backend ogmios_core
balance first
http-request set-path "%[path,regsub(^/api/v0/ogmios/,/)]"
option httpchk GET /health
http-check expect status 200
default-server inter 20s fall 1 rise 2
server local ogmios:1337 check
{{- if eq .Values.cardano_network "mainnet" }}
server koios-ssl api.koios.rest:7443 backup ssl verify none
{{- else }}
server koios-ssl testnet.koios.rest:7443 backup ssl verify none
{{- end }}
backend submitapi
balance first
option httpchk POST /api/submit/tx
http-request set-path "%[path,regsub(^/api/v0/submittx,/api/submit/tx)]"
http-check expect status 415
default-server inter 20s fall 1 rise 2
server local cardano-submit-api:8091 check
{{- if eq .Values.cardano_network "mainnet" }}
server koios-ssl api.koios.rest:443 backup ssl verify none
{{- else }}
server koios-ssl testnet.koios.rest:443 backup ssl verify none
{{- end }}
backend flood_lmt_rate
stick-table type ip size 1m expire 10m store http_req_rate(10s)
backend unauthorized
## Used by monitoring instances only
http-request deny deny_status 401
cache grestcache
total-max-size 1024
max-object-size 51200
process-vary on
max-secondary-entries 500
max-age 300
extraVolumeMounts:
- name: grest
mountPath: /grest
- name: tmp
mountPath: /tmp
extraVolumes:
- name: grest
emptyDir: {}
- configMap:
name: koios-configmap
name: koios-configmap
- name: tmp
emptyDir: {}
image:
repository: haproxytech/haproxy-ubuntu
initContainers:
- name: grest-init
image: gimbalabs/cardano-db-sync-init-container:1.33.0-0
command:
- bash
- -x
- /configmap/haproxy-fetch-grest-poll.sh
env:
- name: CNODE_HOME
value: /grest
- name: KOIOS_BRANCH
value: alpha
volumeMounts:
- mountPath: /configmap
name: koios-configmap
- mountPath: /grest
name: grest
podSecurityPolicy:
create: false
serviceMonitor:
enabled: true