Files not saving to S3-compatible storage
Hello,
I have followed the instructions at https://docs.mayan-edms.com/chapters/object_storage.html to have files stored in Cloudflare R2 (which is S3 compatible) and no matter what I do to the Docker Compose file the files never end up in my bucket. I have edited the environmental variables and still the files are going into the Linux file system.
Any advice? I put my docker-compose.yml below. It should be noted I've tried it both with and without the 'default_acl':'private'.
sergio@mayan:~$ cat docker-compose.yml
version: '3.9'
x-mayan-container:
&mayan-container
env_file: .env
environment:
MAYAN_CELERY_BROKER_URL: amqp://${MAYAN_RABBITMQ_USER:-mayan}:${MAYAN_RABBITMQ_PASSWORD:-mayanrabbitpass}@rabbitmq:5672/${MAYAN_RABBITMQ_VHOST:-mayan}
MAYAN_CELERY_RESULT_BACKEND: redis://:${MAYAN_REDIS_PASSWORD:-mayanredispassword}@redis:6379/1
MAYAN_DATABASES: "{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'${MAYAN_DATABASE_NAME:-mayan}','PASSWORD':'${MAYAN_DATABASE_PASSWORD:-mayandbpass}','USER':'${MAYAN_DATABASE_USER:-mayan}','HOST':'${MAYAN_DATABASE_HOST:-postgresql}'}}"
MAYAN_LOCK_MANAGER_BACKEND: mayan.apps.lock_manager.backends.redis_lock.RedisLock
MAYAN_LOCK_MANAGER_BACKEND_ARGUMENTS: "{'redis_url':'redis://:${MAYAN_REDIS_PASSWORD:-mayanredispassword}@redis:6379/2'}"
MAYAN_PIP_INSTALLS: 'django-storages boto3'
MAYAN_DOCUMENTS_STORAGE_BACKEND: storages.backends.s3boto3.S3Boto3Storage
MAYAN_DOCUMENTS_STORAGE_BACKEND_ARGUMENTS: "{'access_key':'<acces_key>','secret_key':'<secret_key>','bucket_name':'object','default_acl':'private','endpoint_url':'<my_account_id>.r2.cloudflarestorage.com','verify':'True','default_acl':'private'}"
image: ${MAYAN_DOCKER_IMAGE_NAME:-mayanedms/mayanedms}:${MAYAN_DOCKER_IMAGE_TAG:-s4.3}
networks:
- mayan
restart: unless-stopped
volumes:
- ${MAYAN_APP_VOLUME:-app}:/var/lib/mayan
# Optional volumes to access external data like staging or watch folders
# - /opt/staging_folder:/staging_folder
# - /opt/watch_folder:/watch_folder
networks:
mayan:
driver: bridge
# Change to true when using Traefik for increased security.
internal: false
traefik: {}
services:
app:
<<: *mayan-container
profiles:
- all_in_one
# Disable ports if using Traefik.
ports:
- "80:8000"
elasticsearch:
environment:
- bootstrap.memory_lock=true
- discovery.type=single-node
- http.max_content_length=400mb
- xpack.security.enabled=true
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- ELASTIC_PASSWORD=${MAYAN_ELASTICSEARCH_PASSWORD:-mayanespassword}
image: ${MAYAN_DOCKER_ELASTICSEARCH_IMAGE:-docker.elastic.co/elasticsearch/elasticsearch}:${MAYAN_DOCKER_ELASTICSEARCH_TAG:-7.16.0}
networks:
- mayan
# Enable to allow external access to the database.
# ports:
# - "9200:9200"
profiles:
- elasticsearch
restart: unless-stopped
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- ${MAYAN_ELASTICSEARCH_VOLUME:-elasticsearch}:/usr/share/elasticsearch/data
postgresql:
command:
- "postgres"
- "-c"
- "checkpoint_completion_target=0.6"
- "-c"
- "default_statistics_target=200"
- "-c"
- "maintenance_work_mem=128MB"
- "-c"
- "max_connections=150"
- "-c"
- "shared_buffers=256MB"
- "-c"
- "work_mem=8MB"
environment:
POSTGRES_DB: ${MAYAN_DATABASE_NAME:-mayan}
POSTGRES_PASSWORD: ${MAYAN_DATABASE_PASSWORD:-mayandbpass}
POSTGRES_USER: ${MAYAN_DATABASE_USER:-mayan}
image: postgres:${MAYAN_DOCKER_POSTGRES_TAG:-12.10-alpine}
networks:
- mayan
# Enable to allow external access to the database.
# ports:
# - "5432:5432"
profiles:
- postgresql
restart: unless-stopped
volumes:
- ${MAYAN_POSTGRES_VOLUME:-postgres}:/var/lib/postgresql/data
redis:
command:
- redis-server
- --appendonly
- "no"
- --databases
- "3"
- --maxmemory
- "100mb"
- --maxclients
- "500"
- --maxmemory-policy
- "allkeys-lru"
- --save
- ""
- --tcp-backlog
- "256"
- --requirepass
- "${MAYAN_REDIS_PASSWORD:-mayanredispassword}"
image: redis:${MAYAN_DOCKER_REDIS_TAG:-6.2-alpine}
networks:
- mayan
profiles:
- redis
restart: unless-stopped
volumes:
- ${MAYAN_REDIS_VOLUME:-redis}:/data
# Run a frontend gunicorn container
frontend:
<<: *mayan-container
command:
- run_frontend
labels:
- "traefik.enable=true"
- "traefik.http.middlewares.mayan_redirect.redirectscheme.scheme=https"
- "traefik.http.middlewares.mayan_redirect.redirectscheme.permanent=false"
- "traefik.http.routers.mayan_frontend.rule=Host(`${MAYAN_TRAEFIK_EXTERNAL_DOMAIN}`)"
- "traefik.http.routers.mayan_frontend.entrypoints=http"
- "traefik.http.routers.mayan_frontend.middlewares=mayan_redirect"
- "traefik.http.routers.mayan_frontend_https.rule=Host(`${MAYAN_TRAEFIK_EXTERNAL_DOMAIN}`)"
- "traefik.http.routers.mayan_frontend_https.entrypoints=https"
- "traefik.http.routers.mayan_frontend_https.tls=true"
- "traefik.http.routers.mayan_frontend_https.tls.certresolver=letsencrypt"
- "traefik.http.services.mayan_frontend.loadbalancer.server.port=8000"
# Disable ports if using Traefik.
ports:
- "80:8000"
profiles:
- extra_frontend
# Enable to run standalone workers
mountindex:
<<: *mayan-container
cap_add:
- SYS_ADMIN
devices:
- "/dev/fuse:/dev/fuse"
entrypoint:
- /bin/sh
- -c
- 'mkdir --parents /mnt/index && chown mayan:mayan /mnt/index && /usr/local/bin/entrypoint.sh run_command "mountindex --allow-other creation_date /mnt/index"' # Replace "creation_date" with the index of your choice.
profiles:
- mountindex
security_opt:
- apparmor:unconfined
volumes:
- type: bind
source: /mnt/mayan_indexes/creation_date # Host location where the index will show up.
target: /mnt/index # Location inside the container where the index will be mounted. Must the same is in the "entrypoint" section.
bind:
propagation: shared
# Run a separate class A worker
worker_a:
<<: *mayan-container
command:
- run_worker
- worker_a
- "--prefetch-multiplier=1"
profiles:
- extra_worker_a
# Run a separate class B worker
worker_b:
<<: *mayan-container
command:
- run_worker
- worker_b
- "--prefetch-multiplier=1"
profiles:
- extra_worker_b
# Run a separate class C worker
worker_c:
<<: *mayan-container
command:
- run_worker
- worker_c
- "--prefetch-multiplier=1"
profiles:
- extra_worker_c
# Run a separate class D worker
worker_d:
<<: *mayan-container
command:
- run_worker
- worker_d
- "--concurrency=1 --prefetch-multiplier=1"
profiles:
- extra_worker_d
worker_custom_queue:
<<: *mayan-container
command:
- /bin/sh
- -c
- 'MAYAN_QUEUE_LIST=${MAYAN_WORKER_CUSTOM_QUEUE_LIST} /usr/local/bin/run_worker.sh --prefetch-multiplier=1'
profiles:
- extra_worker_custom
# Run a separate Celery beat container
celery_beat:
<<: *mayan-container
command:
- run_celery
- "beat --pidfile= --loglevel=ERROR"
profiles:
- extra_celery_beat
setup_or_upgrade:
<<: *mayan-container
command:
- run_initial_setup_or_perform_upgrade
profiles:
- extra_setup_or_upgrade
restart: "no"
rabbitmq:
image: rabbitmq:${MAYAN_DOCKER_RABBITMQ_TAG:-3.10-management-alpine}
environment:
RABBITMQ_DEFAULT_USER: ${MAYAN_RABBITMQ_USER:-mayan}
RABBITMQ_DEFAULT_PASS: ${MAYAN_RABBITMQ_PASSWORD:-mayanrabbitpass}
RABBITMQ_DEFAULT_VHOST: ${MAYAN_RABBITMQ_VHOST:-mayan}
labels:
- "traefik.enable=${MAYAN_TRAEFIK_RABBITMQ_ENABLE:-false}"
- "traefik.http.routers.mayan_frontend_https.tls=true"
- "traefik.http.routers.mayan_frontend.rule=Host(`${MAYAN_TRAEFIK_EXTERNAL_DOMAIN}`)"
- "traefik.http.routers.mayan_frontend.entrypoints=:15672"
networks:
- mayan
# Enable to allow access to the administration interface.
# ports:
# - "15672:15672"
profiles:
- rabbitmq
restart: unless-stopped
volumes:
- ${MAYAN_RABBITMQ_VOLUME:-rabbitmq}:/var/lib/rabbitmq
traefik:
container_name: "traefik"
command:
# - "--log.level=DEBUG"
- "--api.insecure=${MAYAN_TRAEFIK_API_INSECURE:-false}"
- "--certificatesresolvers.letsencrypt.acme.caserver=${MAYAN_TRAEFIK_LETS_ENCRYPT_SERVER:-https://acme-staging-v02.api.letsencrypt.org/directory}"
- "--certificatesresolvers.letsencrypt.acme.email=${MAYAN_TRAEFIK_LETS_ENCRYPT_EMAIL}"
- "--certificatesresolvers.letsencrypt.acme.storage=/traefik-certificates-letsencrypt/acme.json"
- "--certificatesresolvers.letsencrypt.acme.tlschallenge=${MAYAN_TRAEFIK_LETS_ENCRYPT_TLS_CHALLENGE:-false}"
- "--entrypoints.http.address=:80"
- "--entrypoints.https.address=:443"
- "--providers.docker=true"
- "--providers.docker.exposedbydefault=false"
image: traefik:${MAYAN_DOCKER_TRAEFIK_TAG:-v2.5}
networks:
- mayan
- traefik
ports:
- "80:80"
- "443:443"
- "8080:8080"
profiles:
- traefik
restart: unless-stopped
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
- traefik-certificates-letsencrypt:/traefik-certificates-letsencrypt
volumes:
app:
elasticsearch:
postgres:
mountindex:
rabbitmq:
redis:
traefik-certificates-letsencrypt:```