Skip to content
Commits on Source (262)
......@@ -38,3 +38,4 @@ docs/_templates/layout.html
docs/_templates/footer.html
gitlab-ci-volume/
.DS_Store
config-local.env
This diff is collapsed.
......@@ -2,325 +2,343 @@
host = https://www.transifex.com
lang_map = zh-Hans: zh_Hans
[mayan-edms.acls-4-3]
[mayan-edms.acls-4-4]
file_filter = mayan/apps/acls/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/acls/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.announcements-4-3]
[mayan-edms.announcements-4-4]
file_filter = mayan/apps/announcements/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/announcements/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.appearance-4-3]
[mayan-edms.appearance-4-4]
file_filter = mayan/apps/appearance/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/appearance/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.authentication-4-3]
[mayan-edms.authentication-4-4]
file_filter = mayan/apps/authentication/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/authentication/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.authentication_otp-4-3]
[mayan-edms.authentication_oidc-4-4]
file_filter = mayan/apps/authentication_oidc/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/authentication_oidc/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.authentication_otp-4-4]
file_filter = mayan/apps/authentication_otp/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/authentication_otp/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.autoadmin-4-3]
[mayan-edms.autoadmin-4-4]
file_filter = mayan/apps/autoadmin/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/autoadmin/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.cabinets-4-3]
[mayan-edms.cabinets-4-4]
file_filter = mayan/apps/cabinets/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/cabinets/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.checkouts-4-3]
[mayan-edms.checkouts-4-4]
file_filter = mayan/apps/checkouts/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/checkouts/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.common-4-3]
[mayan-edms.common-4-4]
file_filter = mayan/apps/common/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/common/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.converter-4-3]
[mayan-edms.converter-4-4]
file_filter = mayan/apps/converter/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/converter/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.dashboards-4-3]
[mayan-edms.dashboards-4-4]
file_filter = mayan/apps/dashboards/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/dashboards/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.databases-4-3]
[mayan-edms.databases-4-4]
file_filter = mayan/apps/databases/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/databases/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.dependencies-4-3]
[mayan-edms.dependencies-4-4]
file_filter = mayan/apps/dependencies/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/dependencies/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.django_gpg-4-3]
[mayan-edms.django_gpg-4-4]
file_filter = mayan/apps/django_gpg/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/django_gpg/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.document_comments-4-3]
[mayan-edms.document_comments-4-4]
file_filter = mayan/apps/document_comments/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/document_comments/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.document_indexing-4-3]
[mayan-edms.document_downloads-4-4]
file_filter = mayan/apps/document_downloads/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/document_downloads/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.document_exports-4-4]
file_filter = mayan/apps/document_exports/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/document_exports/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.document_indexing-4-4]
file_filter = mayan/apps/document_indexing/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/document_indexing/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.document_parsing-4-3]
[mayan-edms.document_parsing-4-4]
file_filter = mayan/apps/document_parsing/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/document_parsing/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.document_signatures-4-3]
[mayan-edms.document_signatures-4-4]
file_filter = mayan/apps/document_signatures/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/document_signatures/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.document_states-4-3]
[mayan-edms.document_states-4-4]
file_filter = mayan/apps/document_states/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/document_states/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.documents-4-3]
[mayan-edms.documents-4-4]
file_filter = mayan/apps/documents/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/documents/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.duplicates-4-3]
[mayan-edms.duplicates-4-4]
file_filter = mayan/apps/duplicates/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/duplicates/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.dynamic_search-4-3]
[mayan-edms.dynamic_search-4-4]
file_filter = mayan/apps/dynamic_search/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/dynamic_search/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.events-4-3]
[mayan-edms.events-4-4]
file_filter = mayan/apps/events/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/events/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.file_caching-4-3]
[mayan-edms.file_caching-4-4]
file_filter = mayan/apps/file_caching/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/file_caching/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.file_metadata-4-3]
[mayan-edms.file_metadata-4-4]
file_filter = mayan/apps/file_metadata/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/file_metadata/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.linking-4-3]
[mayan-edms.linking-4-4]
file_filter = mayan/apps/linking/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/linking/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.locales-4-3]
[mayan-edms.locales-4-4]
file_filter = mayan/apps/locales/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/locales/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.lock_manager-4-3]
[mayan-edms.lock_manager-4-4]
file_filter = mayan/apps/lock_manager/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/lock_manager/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.logging-4-3]
[mayan-edms.logging-4-4]
file_filter = mayan/apps/logging/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/logging/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.mailer-4-3]
[mayan-edms.mailer-4-4]
file_filter = mayan/apps/mailer/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/mailer/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.mayan_statistics-4-3]
[mayan-edms.mayan_statistics-4-4]
file_filter = mayan/apps/mayan_statistics/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/mayan_statistics/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.messaging-4-3]
[mayan-edms.messaging-4-4]
file_filter = mayan/apps/messaging/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/messaging/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.metadata-4-3]
[mayan-edms.metadata-4-4]
file_filter = mayan/apps/metadata/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/metadata/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.mime_types-4-3]
[mayan-edms.mime_types-4-4]
file_filter = mayan/apps/mime_types/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/mime_types/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.mirroring-4-3]
[mayan-edms.mirroring-4-4]
file_filter = mayan/apps/mirroring/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/mirroring/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.navigation-4-3]
[mayan-edms.navigation-4-4]
file_filter = mayan/apps/navigation/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/navigation/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.ocr-4-3]
[mayan-edms.ocr-4-4]
file_filter = mayan/apps/ocr/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/ocr/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.organizations-4-3]
[mayan-edms.organizations-4-4]
file_filter = mayan/apps/organizations/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/organizations/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.permissions-4-3]
[mayan-edms.permissions-4-4]
file_filter = mayan/apps/permissions/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/permissions/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.platform-4-3]
[mayan-edms.platform-4-4]
file_filter = mayan/apps/platform/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/platform/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.quotas-4-3]
[mayan-edms.quotas-4-4]
file_filter = mayan/apps/quotas/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/quotas/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.redactions-4-3]
[mayan-edms.redactions-4-4]
file_filter = mayan/apps/redactions/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/redactions/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.rest_api-4-3]
[mayan-edms.rest_api-4-4]
file_filter = mayan/apps/rest_api/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/rest_api/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.signature_captures-4-3]
[mayan-edms.signature_captures-4-4]
file_filter = mayan/apps/signature_captures/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/signature_captures/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.smart_settings-4-3]
[mayan-edms.smart_settings-4-4]
file_filter = mayan/apps/smart_settings/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/smart_settings/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.sources-4-3]
[mayan-edms.sources-4-4]
file_filter = mayan/apps/sources/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/sources/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.storage-4-3]
[mayan-edms.storage-4-4]
file_filter = mayan/apps/storage/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/storage/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.tags-4-3]
[mayan-edms.tags-4-4]
file_filter = mayan/apps/tags/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/tags/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.task_manager-4-3]
[mayan-edms.task_manager-4-4]
file_filter = mayan/apps/task_manager/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/task_manager/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.templating-4-3]
[mayan-edms.templating-4-4]
file_filter = mayan/apps/templating/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/templating/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.testing-4-3]
[mayan-edms.testing-4-4]
file_filter = mayan/apps/testing/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/testing/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.user_management-4-3]
[mayan-edms.user_management-4-4]
file_filter = mayan/apps/user_management/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/user_management/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.views-4-3]
[mayan-edms.views-4-4]
file_filter = mayan/apps/views/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/views/locale/en/LC_MESSAGES/django.po
source_lang = en
type = PO
[mayan-edms.web_links-4-3]
[mayan-edms.web_links-4-4]
file_filter = mayan/apps/web_links/locale/<lang>/LC_MESSAGES/django.po
source_file = mayan/apps/web_links/locale/en/LC_MESSAGES/django.po
source_lang = en
......
4.4.3 (2023-02-11)
==================
- Improve transformation views to always pass the object
having the transformation applied.
- Add support to the ``Link`` class for dynamic view keyword arguments, icon,
resolved object, and permissions.
- Update the transformation, decorations, and redactions links to use
dynamic view keyword arguments, icons, resolved objects, and permissions.
- Move transformation and redactions links to either their own ``links.py``
module. In the case of the ``documents`` app, the module is named
``miscellaneous_links.py``.
- Improve permissions handling of the transformation, decorations, and
redactions links.
- Improve transformation and redaction link testing.
- Sanitize tag labels to avoid XSS abuse (CVE-2022-47419: Mayan EDMS Tag XSS).
This is a limited scope weakness of the tagging system markup that can be
used to display an arbitrary text when selecting a tag for attachment to
or removal from a document.
It is not possible to circumvent Mayan EDMS access control system or
expose arbitrary information with this weakness.
Attempting to exploit this weakness requires a privileged account and
is not possible to enable from a guest or an anonymous account. Visitors
to a Mayan EDMS installation cannot exploit this weakness.
Any usage of this weakness remains logged in the event system making
it easy to track down any bad actors.
Due to all these factors, the surface of attack of this weakness is
very limited, if any.
There are no known actual or theoretical attacks exploiting this
weakness to expose or destroy data.
- Drop support for Python 3.7 and Python 3.8. Python 3.9 is now the minimum
version supported. This change happened in version 4.4 but was not
documented. Closes GitLab issue #1137. Thanks to joh-ku (@joh-ku)
for the report and research.
4.4.2 (2023-01-23)
==================
- Merge request !106. Do not show server communication modal
for interrupted AJAX requests. Thanks to
Nicholas Buttigieg (@nicholasbuttigieg) and
Kyle Pullicino (@KPull) for the patch.
4.4.1 (2023-01-19)
==================
- Fix list filtering template issue caused by caching.
- GitOps updates:
- Add makefile targets to trigger standalone builds.
- Increase artifact expiration.
- Add PIP and APT caching to documentation and python build stages.
- Add GitLab CI job dependencies.
- Reuse Python build in stages.
- Convert branches into literals.
- Remove duplicated code in jobs.
- Simplify installation documentation.
4.4 (2023-01-16)
================
- Update Docker image tags:
- Docker from 20-dind to 20.10.21-dind
- ElasticSearch from 7.17.0 to 7.17.7
- PostgreSQL from 12.11-alpine to 13.11.2-alpine
- Redis from 6.2-alpine to 7.0.5-alpine
- Update dependencies versions:
- Celery from 5.1.2 to 5.2.7.
- extract msg from 0.34.3 to 0.36.4.
- djangorestframework from 3.13.1 to 3.14.0.
- drf-yasg from 1.20.0 to 1.21.4.
- sentry-sdk from 1.5.8 to 1.21.1.
- Pillow from 9.2.0 to 9.4.0.
- Increase compatibility of the file caching storage usage with more S3
object storage implementations.
- Add support for OpenID Connect (OIDC) authentication. Adds the new
``authentication_oidc`` app.
- Add the parent cabinet as the action object to the cabinet creation event
when a child cabinet is created.
- Add the cabinet deleted event. This event is committed when a child cabinet
is deleted. The parent cabinet is recorded as the action object for the
event.
- Fix the function interfaces when calling ``get_mayan_object_permissions``
to ``get_mayan_view_permissions`` to override an API view permission
layout.
- Update navigation permission check to short circuit check when the
current user has not logged in yet.
- Cabinet updates:
- Use the same permission layout to create parent and child cabinets from
the API as from the HTTP views.
- The create permission is now required to create parent as well as child
cabinets. This change replaces requiring the edit permission to create
child cabinets via the HTTP views.
- Data from file and download content creation or examination now defaults
to byte format instead of unicode.
- Include Django Storages and boto3 Python libraries by default.
- Use the optimized version (``+=``) of the ``+`` operator.
- Moved the document version export code to its own app called
``document_exports``. Existing export events and permissions are
migrated automatically.
- Improved invalid permission error handling. Instead of returning an error
that stop execution when an invalid permission identifier is requested,
the permission model will return a one line text indicating that the
permission name is invalid. This error message will be displayed in place
of the intended permission label.
This behavior was also extended to cover invalid permission
namespace requests.
A troubleshooting section is added explaining the possible
reasons for the and the solution.
- Search refactor:
- Added search syntax pre processor and convert all backend to work with
it.
- Simplify scoped search syntax.
- Allow more than two operands per operator.
- Implement NOT operator.
- Improve AND and OR operators.
- Split search classes into separate modules.
- Add search field subclasses.
- Move instance value retrieval to search fields.
- Add virtual fields.
- Add an "ANY field" virtual field.
- All search is now conducted using an internal scope system.
- Move search syntax decoding to its own class and subclasses named
``SearchInterpreter``.
- Move the search bar to the main menu top bar.
- Add support for data typing.
- Normalize data during index and search.
- Testing improvement. Track test document IDs. Keep a list of the test
document IDs in number and string format.
- New reusable view mixin, ``MultipleExternalObjectViewMixin``.
- Add Hebrew to the default list of document languages.
- Enable Docker BuildKit.
- Add dedicated Docker build RUN cache.
- Docker Compose file changes:
- Configurable frontend HTTP port via the .env file.
- Unify frontend and all_in_one profiles HTTP and Traefik configuration.
- Support Let's Entry TLS termination for all_in_one profile.
- Configurable RabbitMQ administration HTTP port via the .env file.
- Configurable Traefik dashboard, HTTP and HTTPS entrypoints ports via
the .env file.
- Configurable Traefik Let's Encrypt certificate volume location.
- Support Let's Encrypt DNS challenge.
- Isolate compressed file MIME type matchin exception catching to the
pertinent code.
- Download file updates:
- Associate download files to a specific users.
- Add delete, download, and view permissions.
- Add download file size column.
- Support bulk document file downloads.
- Move document file download code to the new document downloads app. Migrate
existing document file download permission and events.
- Permission updates:
- Improve permission caching. Remove custom caching code and
use upstream Django caching utilities.
- Rename variables for clarity.
- Update dependent code to match class interface changes.
- Short circuit the source column source object resolution code to support
the list template showing columns even on empty lists. Unlike models and
queryset, empty lists of class instances won't display any columns.
- Download GPG keys in binary mode.
- Show more details when a bootstrap setting parsing error occurs.
- Add libfuse2, libsasl2-dev, and libldap2-dev to the development setup
makefile target.
- Convert pagination template into a partial.
- Add .msg file metadata drivers.
- Convert the metadata value and the file metadata value fields from
character fields with a maximum length of 255 characters to text fields.
- Improve the settings apps navigation.
- Collapse action menus by default.
- Split links in the list items template into action links and view (facet)
links.
- Remove Docker mirror configuration from the GitLab CI file. This is up to
the runner to configure.
- Show cache partitions and partition file totals. This helps determine how
effective is a cache maximum size value by showing how many objects and
files the cache size limit is able to yield.
- Statistics updates:
- Improve statistics navigation.
- Add doughnut chart type statistic.
- Add pie chart type statistic.
- Add column displaying the chart type per statistic.
- Update chartjs from version 2.8.0 to 3.9.1.
- Unify chart templates.
- Autoload statistics modules.
- Unify ChartJS templates.
- Support passing full chart context not just plot data.
- Fix app URL layout.
- Fix app URL typo.
- Fix statistic queue view navigation context.
- Add three document pie chart statistics: document count per document type,
document file count per document type, document file page count per
document type.
- Add documentation directives to insert setting or setting namespace
instances.
- Improve search and object storage documentation by adding automated
setting references.
- Add third state to column sorting. The sorting states are now: ascending,
descending, none.
- Support sorting multiple columns.
- Add a permission count column to ACLs.
- Add support for setting choices.
- Add an HTML to better format setting values.
- Fix search again view redirect. Retain the query from the previous view.
- Rename the "Search" facet link to "Basic search".
- Fix "Match All" behavior when using the "Search again" link.
- Replace "Match All" field with a radio box to allow supporting search
again persistence and also the default state of the field.
- Convert the "Match All" field name into a literal.
- Split the document file creation method into smaller units. This reduces
the complexity of the several conditional statements.
- Ensure the document file is created even if there are errors during the
uploaded file introspection.
- Encapsulate mozilla-django-oidc settings as Mayan authentication backend
arguments.
- Add support for OpenID Connect Discovery (https://openid.net/specs/openid-connect-discovery-1_0.html).
- Add Keycloak Docker Compose service.
- Make Docker Compose service image name configurable.
- User interface updates:
- Collapse views and actions by default.
- Add collapsed views and actions icons.
- Move views and actions markup to their own respective partial templates.
This reduces duplication and improves usability of the markup.
- Lower the z-index of the sidebar to avoid menus to display behind it.
- Add simulated horizontal rulers to the body of tables.
- Support slim dropdown menus.
- Move navigation to the card footer.
- Fix click events passing through views and action dropdown caret icons.
- Make the words "Actions" and "Views" translatable.
- Add two new table columns. One columns for views and another the actions.
- Unroll single action menus to a button.
- Reduce table padding to increase data area.
- Minor spacing and margin tweaks.
- Code style refactor and cleanup:
- Strip trailing commas.
- Sort arguments, dictionary keys and class methods.
- Unroll nested contexts.
- Separate model data and business logic code.
- Move add or remove code to models. Directly and as added methods to
external models.
- Pass the user to action methods instead of injecting the user as the
event actor. Injecting the user as the event actor will be done only
on immediate methods that do not allow arguments or data layer model
methods with well defined upstream arguments.
- Add keyword arguments.
- Rename mixins modules to be more explicit.
- Normalize the ``UploadWizard`` class ``step_post_upload_process`` method
arguments.
- Remove many instances of ``force_text``.
- Move several ``upload_to`` functions to their corresponding app's
``utils`` module.
- Promote private ``_user`` argument to an official argument.
- API views refactor:
- Remove injected objects on API views. Each API view needs to query the
object explicitly. This is change is less efficient but was made to
mirror how upstream DRF works.
- Pass the view object to the action object API view.
- Add labels to serializer fields.
- Track the user when purging caches and cache partitions.
- Create a new permission to change the type of a document.
When support for changing the type of a document was added, it was
considered a property and controlled via the document property edit
permission.
Since changing the type of a documents now causes a cascade of other
changes, it was isolated as an individual class of event along
with its own permission.
The new document change type permission is required for the document being
changed and for the document type to which the document will be changed
into.
- Update the file metadata model ``verbose_name`` attribute to be the
``help_text`` attribute.
- Update the document parsing ``verbose_name`` attribute to be the
``help_text`` attribute.
- Update the document version OCR ``verbose_name`` attribute to be the
``help_text`` attribute.
- Update the search API to provide a dummy model serializer during Swagger
introspection.
- Update the sources actions API to provide a dummy serializer during Swagger
introspection.
- Fix Swagger schema model definition introspection. Updated REST API views
to behave like user interface views and returning querysets either via
the ``source_queryset`` property or the ``get_source_queryset`` method.
This prevents API views from overriding the queryset return methods and
allows the ``SchemaInspectionAPIViewMixin`` mixin to work in all
instances.
- Add support for platform client backends to register tool links.
- Lower the default Sentry client sample rate from 0.05 to 0.01.
- Add new setting to disable automatic upload after dragging files to the
DropZone widget. The setting is named ``VIEWS_SHOW_DROPZONE_SUBMIT_BUTTON``
and defaults to ``False``.
- Raise an ``ImproperlyConfigured`` exception when a model is registered for
error logging more than once.
- Move error logging registration of document models to the documents app.
- OCR updates:
- Move error logging from the document version to the document version
page.
- Add OCR backend ``_execute`` to avoid subclasses from calling the super
class.
- The base class now prepares the image to be processed and passes the
file object to the subclass.
- Move OCR finished event commit from the task to the manager.
- Restore object event attributes when the event is ignored.
- Error log registration now register error log permissions too by default.
- Improve base settings initialization:
- Replace ``os.path`` with ``pathlib.Path`` to do path manipulation.
- Use the default secret key value only if the secret key file is not
found.
- Don't obscure errors when reading the secret key file.
- Remove the ``home_view`` setting from the default Template context.
Template instances need to include their own context using the new
``context`` argument.
- Add templating support to bootstrap settings. Template names are the same
as the bootstrap setting but include the ``SETTING_TEMPLATE_`` prefix.
Environment variables, Python global and config file values are available
to the template.
- Remove deprecated management commands:
- ``checkdependencies`` replaced by ``dependencies_check``.
- ``checkversion`` replaced by ``dependencies_check_version``.
- ``createautoadmin`` replaced by ``autoadmin_create``.
- ``generaterequirements`` replaced by ``dependencies_generate_requirements``.
- ``initialsetup`` replaced by ``common_initial_setup``.
- ``installdependencies`` replaced by ``dependencies_install``.
- ``mountindex`` replaced by ``mirroring_mount_index``.
- ``performupgrade`` replaced by ``common_perform_upgrade``.
- ``platformtemplate`` replaced by ``platform_template``.
- ``preparestatic`` replaced by ``appearance_prepare_static``.
- ``purgelocks`` replaced by ``lock_manager_purge_locks``.
- ``purgepermissions`` replaced by ``permissions_purge``.
- ``purgeperiodictasks`` replaced by ``task_manager_purge_periodic_tasks``.
- ``purgestatistics`` replaced by ``statistics_purge``.
- ``revertsettings`` replaced by ``settings_revert``.
- ``savesettings`` replaced by ``settings_save``.
- ``showsettings`` replaced by ``settings_show``.
- ``showversion`` replaced by ``dependencies_show_version``.
- Update the makefile to enable the Sentry client if the ``SENTRY_DSN``
value is passed to the ``runserver``, ``runserver-plus``, or
``staging-frontend`` targets.
- Add new setting to disable logging message ANSI color codes. The setting
is named ``LOGGING_DISABLE_COLOR_FORMATTER`` and defaults to ``False``.
- Standardize management command testing.
- Move management command names to the ``literals`` module of each app.
- GitOps updates:
- Add configurable remote branch for GitOps.
- Support a local environment config file names ``config-local.env``.
This file is ignored by Git and meant to override values of ``config.env``.
4.3.5 (2023-01-10)
==================
- Fix error when deleting a user form the user interface. Closes GitLab
issue #1125. Thanks to friki67 (@friki67) for the report and
Jan Przychodniak (@janprzychodniak) for the additional debug information.
4.3.4 (2022-12-19)
==================
- Merge fixes from version 4.2.13.
- Fix reference to ``ocr_errors`` in
``mayan.apps.ocr.tasks.task_document_version_ocr_finished``. Closes GitLab
issue #1131. Thanks to Olivier D. (@odelseth) for the report and debug
information.
- Fix click events passing through views and action dropdown caret icons.
Activating the dropdown menu by clicking on the menu's caret no longer
select the document file or version card.
4.3.3 (2022-11-15)
==================
- Fixes from version 4.2.12.
......@@ -14,7 +412,7 @@
are only downloaded from authoritative locations and each dependency is
pinned to a specific version to guarantee immutable releases.
Due to all these factors surface of attack of this vulnerability is
Due to all these factors, surface of attack of this vulnerability is
very limited for older versions of Mayan EDMS, it is also very improbable,
very difficulty to accomplish and very difficult to remain undetected.
......@@ -87,7 +485,10 @@
- Support AJAX request throttling. Prevents users from requesting too many
consecutive page loads. Defaults to a maximum of 10 requests in 5 seconds
of less. This applies only to the user interface. The AJAX throttling
resets the moment the last pending AJAX request is completed.
resets when a pending request is completed. Added the settings
``APPEARANCE_THROTTLING_MAXIMUM_REQUESTS`` and
``APPEARANCE_THROTTLING_TIMEOUT``. Display a message notifying users when
throttling is in effect.
- ``BaseBackend`` class improvements.
- Selectable identifier via the ``_backend_identifier`` property. Defaults
......@@ -478,6 +879,19 @@
- Display a warning message in the setting edit view when local storage is
disabled.
4.2.13 (2022-12-18)
===================
- Fix document file and document version print form submit button.
- Fix tagged document list view permission filtering. The permission
layout remains the same. Only the method in which the permissions is
checked was updated.
- Fix metadata add action actor assignment. This assignment is not
currently used by either the view or the API which assign the actor
directly themselves.
- Silence Docker Compose warning "MAYAN_WORKER_CUSTOM_QUEUE_LIST variable
is not set". Closes GitLab issue #1129. Thanks to GR Buck (@graybuck)
for the report.
4.2.12 (2022-11-13)
===================
- Fixes from version 4.1.10.
......
#!make
include config.env
ifneq ($(wildcard config-local.env),)
include config-local.env
endif
ifndef MODULE
override MODULE = --mayan-apps
endif
......@@ -13,6 +17,12 @@ ifndef SETTINGS
override SETTINGS = mayan.settings.testing.development
endif
SENTRY_COMMAND = \
if [ $(SENTRY_DSN) ]; then \
export MAYAN_PLATFORM_CLIENT_BACKEND_ENABLED='["mayan.apps.platform.client_backends.ClientBackendSentry"]'; \
export MAYAN_PLATFORM_CLIENT_BACKEND_ARGUMENTS='{"mayan.apps.platform.client_backends.ClientBackendSentry":{"dsn":"$(SENTRY_DSN)","environment":"development"}}'; \
fi
TEST_COMMAND = ./manage.py test $(MODULE) --settings=$(SETTINGS) $(SKIPMIGRATIONS) $(DEBUG) $(ARGUMENTS)
TEST_ELASTIC_CONTAINER_NAME = mayan-test-elastic
......@@ -112,18 +122,6 @@ test-all-migrations-with-postgresql:
export MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'$(DEFAULT_DATABASE_NAME)','PASSWORD':'$(DEFAULT_DATABASE_PASSWORD)','USER':'$(DEFAULT_DATABASE_USER)','HOST':'127.0.0.1'}}"; \
./manage.py test --mayan-apps --settings=mayan.settings.testing.development --no-exclude --tag=migration_test
gitlab-ci-update: ## Update the GitLab CI file from the platform template.
gitlab-ci-update: copy-config-env
./manage.py platformtemplate gitlab-ci > .gitlab-ci.yml
gitlab-ci-run: ## Execute a GitLab CI job locally
gitlab-ci-run:
if [ -z $(GITLAB_CI_JOB) ]; then echo "Specify the job to execute using GITLAB_CI_JOB."; exit 1; fi; \
docker rm --force gitlab-runner || true
docker run --detach --name gitlab-runner --restart no --volume $$PWD:$$PWD --volume /var/run/docker.sock:/var/run/docker.sock gitlab/gitlab-runner:latest
docker exec --interactive --tty --workdir $$PWD gitlab-runner gitlab-runner exec docker --docker-privileged --docker-volumes /var/run/docker.sock:/var/run/docker.sock --docker-volumes $$PWD/gitlab-ci-volume:/builds $(GITLAB_CI_JOB)
docker rm --force gitlab-runner || true
# Coverage
coverage-run: ## Run all tests and measure code execution.
......@@ -265,7 +263,7 @@ python-sdist-test-suit: python-sdist
sh -c '\
. _virtualenv/bin/activate; \
pip install `ls dist/*.gz`; \
_virtualenv/bin/mayan-edms.py initialsetup; \
_virtualenv/bin/mayan-edms.py common_initial_setup; \
pip install --requirement requirements/testing.txt; \
_virtualenv/bin/mayan-edms.py test --mayan-apps \
'
......@@ -277,7 +275,7 @@ python-wheel-test-suit: wheel
sh -c '\
. _virtualenv/bin/activate; \
pip install `ls dist/*.whl`; \
_virtualenv/bin/mayan-edms.py initialsetup; \
_virtualenv/bin/mayan-edms.py common_initial_setup; \
pip install mock==2.0.0; \
_virtualenv/bin/mayan-edms.py test --mayan-apps \
'
......@@ -288,89 +286,13 @@ generate-setup: generate-requirements
@echo "Complete."
generate-requirements: ## Generate all requirements files from the project depedency declarations.
@./manage.py generaterequirements build > requirements/build.txt
@./manage.py generaterequirements development > requirements/development.txt
@./manage.py generaterequirements documentation > requirements/documentation.txt
@./manage.py generaterequirements documentation_override > requirements/documentation_override.txt
@./manage.py generaterequirements testing > requirements/testing-base.txt
@./manage.py generaterequirements production --exclude=django > requirements/base.txt
@./manage.py generaterequirements production --only=django > requirements/common.txt
# Major releases
gitlab-release-documentation: ## Trigger the documentation build and publication using GitLab CI
gitlab-release-documentation:
git push
git push --tags
git push origin :releases/documentation || true
git push origin HEAD:releases/documentation
gitlab-release-docker-major: ## Trigger the Docker image build and publication using GitLab CI
gitlab-release-docker-major:
git push
git push --tags
git push origin :releases/docker_major || true
git push origin HEAD:releases/docker_major
gitlab-release-python-major: ## Trigger the Python package build and publication using GitLab CI
gitlab-release-python-major:
git push
git push --tags
git push origin :releases/python_major || true
git push origin HEAD:releases/python_major
gitlab-release-all-major: ## Trigger the Python package, Docker image, and documentation build and publication using GitLab CI
gitlab-release-all-major:
git push
git push --tags
git push origin :releases/all_major || true
git push origin HEAD:releases/all_major
# Minor releases
gitlab-release-docker-minor: ## Trigger the Docker image build and publication of a minor version using GitLab CI
gitlab-release-docker-minor:
git push
git push --tags
git push origin :releases/docker_minor || true
git push origin HEAD:releases/docker_minor
gitlab-release-python-minor: ## Trigger the Python package build and publication of a minor version using GitLab CI
gitlab-release-python-minor:
git push
git push --tags
git push origin :releases/python_minor || true
git push origin HEAD:releases/python_minor
gitlab-release-all-minor: ## Trigger the Python package, Docker image build and publication of a minor version using GitLab CI
gitlab-release-all-minor:
git push
git push --tags
git push origin :releases/all_minor || true
git push origin HEAD:releases/all_minor
# Internal testing
gitlab-tests-internal-all: ## Trigger all tests as a CD/CI pipeline
gitlab-tests-internal-all:
git push internal
git push internal --tags
git push internal :tests/all || true
git push internal HEAD:tests/all
gitlab-tests-internal-base: ## Trigger normal and migration tests as a CD/CI pipeline
gitlab-tests-internal-base:
git push internal
git push internal --tags
git push internal :tests/base || true
git push internal HEAD:tests/base
gitlab-tests-internal-upgrade: ## Trigger upgrade tests as a CD/CI pipeline
gitlab-tests-internal-upgrade:
git push internal
git push internal --tags
git push internal :tests/upgrade || true
git push internal HEAD:tests/upgrade
@./manage.py dependencies_generate_requirements build > requirements/build.txt
@./manage.py dependencies_generate_requirements development > requirements/development.txt
@./manage.py dependencies_generate_requirements documentation > requirements/documentation.txt
@./manage.py dependencies_generate_requirements documentation_override > requirements/documentation_override.txt
@./manage.py dependencies_generate_requirements testing > requirements/testing-base.txt
@./manage.py dependencies_generate_requirements production --exclude=django > requirements/base.txt
@./manage.py dependencies_generate_requirements production --only=django > requirements/common.txt
# Dev server
......@@ -390,10 +312,10 @@ manage-with-postgresql: ## Run the development server using a Docker PostgreSQL
./manage.py $(filter-out $@,$(MAKECMDGOALS)) --settings=mayan.settings.development
runserver: ## Run the development server.
./manage.py runserver --settings=mayan.settings.development $(ADDRPORT)
$(SENTRY_COMMAND); ./manage.py runserver --settings=mayan.settings.development $(ADDRPORT)
runserver-plus: ## Run the Django extension's development server.
./manage.py runserver_plus --settings=mayan.settings.development $(ADDRPORT)
$(SENTRY_COMMAND); ./manage.py runserver_plus --settings=mayan.settings.development $(ADDRPORT)
shell-plus: ## Run the shell_plus command.
./manage.py shell_plus --settings=mayan.settings.development
......@@ -462,14 +384,14 @@ docker-redis-stop:
staging-start: ## Launch and initialize production-like services using Docker (PostgreSQL and Redis).
staging-start: staging-stop docker-postgresql-start docker-redis-start
export MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'$(DEFAULT_DATABASE_NAME)','PASSWORD':'$(DEFAULT_DATABASE_PASSWORD)','USER':'$(DEFAULT_DATABASE_USER)','HOST':'127.0.0.1'}}"; \
./manage.py initialsetup --settings=mayan.settings.staging.docker
./manage.py common_initial_setup --settings=mayan.settings.staging.docker
staging-stop: ## Stop and delete the Docker production-like services.
staging-stop: docker-postgresql-stop docker-redis-stop
staging-frontend: ## Launch a front end instance that uses the production-like services.
export MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'$(DEFAULT_DATABASE_NAME)','PASSWORD':'$(DEFAULT_DATABASE_PASSWORD)','USER':'$(DEFAULT_DATABASE_USER)','HOST':'127.0.0.1'}}"; \
./manage.py runserver --settings=mayan.settings.staging.docker
$(SENTRY_COMMAND); ./manage.py runserver --settings=mayan.settings.staging.docker
staging-worker: ## Launch a worker instance that uses the production-like services.
export MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'$(DEFAULT_DATABASE_NAME)','PASSWORD':'$(DEFAULT_DATABASE_PASSWORD)','USER':'$(DEFAULT_DATABASE_USER)','HOST':'127.0.0.1'}}"; \
......@@ -514,7 +436,7 @@ setup-dev-environment: setup-dev-operating-system-packages setup-dev-python-libr
setup-dev-operating-system-packages: ## Install the operating system packages needed for development.
setup-dev-operating-system-packages:
sudo apt-get install --yes exiftool gcc gettext gnupg1 graphviz libcairo2 libffi-dev libjpeg-dev libpng-dev poppler-utils python3-dev sane-utils tesseract-ocr-deu
sudo apt-get install --yes exiftool gcc gettext gnupg1 graphviz libcairo2 libffi-dev libfuse2 libjpeg-dev libldap2-dev libpng-dev libsasl2-dev poppler-utils python3-dev sane-utils tesseract-ocr-deu
setup-dev-python-libraries: ## Install the Python libraries needed for development.
setup-dev-python-libraries:
......@@ -548,4 +470,5 @@ devpi-stop:
killall devpi-server || true
-include docker/Makefile
-include gitlab-ci/Makefile
-include vagrant/Makefile
......@@ -23,15 +23,36 @@ DOCKER_HOST_REGISTRY_PORT=5000
# Docker images
DOCKER_IMAGE_MAYAN_NAME=mayanedms/mayanedms
# Docker image versions
DOCKER_DIND_IMAGE_VERSION=docker:20-dind
DOCKER_ELASTIC_IMAGE_VERSION=docker.elastic.co/elasticsearch/elasticsearch:7.17.0
DOCKER_LINUX_IMAGE_VERSION=debian:11.4-slim
DOCKER_KEYCLOAK_IMAGE_VERSION=keycloak/keycloak:20.0.1
DOCKER_DIND_IMAGE_VERSION=docker:20.10.21-dind
DOCKER_ELASTIC_IMAGE_VERSION=elasticsearch:7.17.7
DOCKER_LINUX_IMAGE_VERSION=debian:11.5-slim
DOCKER_MYSQL_IMAGE_VERSION=mysql:8.0
DOCKER_ORACLE_IMAGE_VERSION=wnameless/oracle-xe-11g-r2
DOCKER_POSTGRES_IMAGE_VERSION=postgres:12.11-alpine
DOCKER_POSTGRES_IMAGE_VERSION=postgres:13.8-alpine
DOCKER_PYTHON_IMAGE_VERSION=python:3.10-slim
DOCKER_RABBITMQ_IMAGE_VERSION=rabbitmq:3.10-alpine
DOCKER_REDIS_IMAGE_VERSION=redis:6.2-alpine
DOCKER_RABBITMQ_IMAGE_VERSION=rabbitmq:3.11.2-alpine
DOCKER_REDIS_IMAGE_VERSION=redis:7.0.5-alpine
# GitOps
GIT_REMOTE_NAME=origin
GITLAB_CI_BRANCH_BUILDS_DOCKER=builds/docker
GITLAB_CI_BRANCH_BUILDS_DOCUMENTATION=builds/documentation
GITLAB_CI_BRANCH_BUILDS_PYTHON=builds/python
GITLAB_CI_BRANCH_RELEASES_ALL_MAJOR=releases/all/major
GITLAB_CI_BRANCH_RELEASES_ALL_MINOR=releases/all/minor
GITLAB_CI_BRANCH_RELEASES_DOCKER_MAJOR=releases/docker/major
GITLAB_CI_BRANCH_RELEASES_DOCKER_MINOR=releases/docker/minor
GITLAB_CI_BRANCH_RELEASES_DOCUMENTATION=releases/documentation
GITLAB_CI_BRANCH_RELEASES_NIGHTLY=releases/nightly
GITLAB_CI_BRANCH_RELEASES_PYTHON_MAJOR=releases/python/major
GITLAB_CI_BRANCH_RELEASES_PYTHON_MINOR=releases/python/minor
GITLAB_CI_BRANCH_RELEASES_STAGING=releases/staging
GITLAB_CI_BRANCH_RELEASES_TESTING=releases/testing
GITLAB_CI_BRANCH_TESTS_ALL=tests/all
GITLAB_CI_BRANCH_TESTS_DOCKER=tests/docker
GITLAB_CI_BRANCH_TESTS_PYTHON_ALL=tests/python/all
GITLAB_CI_BRANCH_TESTS_PYTHON_BASE=tests/python/base
GITLAB_CI_BRANCH_TESTS_PYTHON_UPGRADE=tests/python/upgrade
# Gunicorn
GUNICORN_LIMIT_REQUEST_LINE=4094
GUNICORN_MAX_REQUESTS=500
......
......@@ -71,10 +71,10 @@ MAYAN_DATABASES="{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'${
MAYAN_LOCK_MANAGER_BACKEND="mayan.apps.lock_manager.backends.redis_lock.RedisLock" \
MAYAN_LOCK_MANAGER_BACKEND_ARGUMENTS="{'redis_url':'redis://:${REDIS_PASSWORD}@${REDIS_HOST}:6379/2'}" \
MAYAN_MEDIA_ROOT="${MAYAN_MEDIA_ROOT}" \
${MAYAN_INSTALLATION_FOLDER}bin/mayan-edms.py initialsetup
${MAYAN_INSTALLATION_FOLDER}bin/mayan-edms.py common_initial_setup
echo -e "12. Create the supervisor file at /etc/supervisor/conf.d/mayan-edms.conf \n"
sudo -u mayan MAYAN_MEDIA_ROOT="${MAYAN_MEDIA_ROOT}" ${MAYAN_INSTALLATION_FOLDER}bin/mayan-edms.py platformtemplate supervisord | sudo sh -c "cat > /etc/supervisor/conf.d/mayan-edms.conf"
sudo -u mayan MAYAN_MEDIA_ROOT="${MAYAN_MEDIA_ROOT}" ${MAYAN_INSTALLATION_FOLDER}bin/mayan-edms.py platform_template supervisord | sudo sh -c "cat > /etc/supervisor/conf.d/mayan-edms.conf"
echo -e "13. Enable and restart the services \n"
sudo systemctl enable supervisor
......
......@@ -33,6 +33,20 @@ COMPOSE_PROFILES=all_in_one,postgresql,rabbitmq,redis
# MAYAN_RABBITMQ_VHOST=
# MAYAN_REDIS_PASSWORD=
# MAYAN_FRONTEND_HTTP_PORT=80
MAYAN_WORKER_CUSTOM_QUEUE_LIST=
# Change if you use external services.
MAYAN_DOCKER_WAIT="postgresql:5432 rabbitmq:5672 redis:6379"
# RabbitMQ
# MAYAN_RABBITMQ_ADMIN_HTTP_PORT=15672
# Allows running an additional worker with a custom list of queues.
MAYAN_WORKER_CUSTOM_QUEUE_LIST=
# Traefik
# Enable to use production Let's Encrypt server.
......@@ -48,12 +62,32 @@ COMPOSE_PROFILES=all_in_one,postgresql,rabbitmq,redis
MAYAN_TRAEFIK_LETS_ENCRYPT_EMAIL=
MAYAN_TRAEFIK_EXTERNAL_DOMAIN=
# Expose the RabbitMQ administrative interface.
# Expose the Traefik secure dashboard.
MAYAN_TRAEFIK_DASHBOARD_ENABLE=false
# Traefik secure dashboard username and password.
# Obtained using: echo $(htpasswd -nB your_username_of_choice)
# Enclose in single quotes.
MAYAN_TRAEFIK_DASHBOARD_AUTHENTICATION=''
# Insecure value of admin:admin
# MAYAN_TRAEFIK_DASHBOARD_AUTHENTICATION='admin:$$2y$$05$$mr0lbnUpImKmjXyClgdV6OfwKgOFZKq7ldadIcQXHMwzIZI99wos6'
# Expose the frontend through Traefik.
MAYAN_TRAEFIK_FRONTEND_ENABLE=false
# Expose the RabbitMQ administrative interface through Traefik.
MAYAN_TRAEFIK_RABBITMQ_ENABLE=false
# Change if you use external services.
MAYAN_DOCKER_WAIT="postgresql:5432 rabbitmq:5672 redis:6379"
# MAYAN_TRAEFIK_DASHBOARD_HTTP_PORT=8080
# MAYAN_TRAEFIK_LETSENCRYPT_VOLUME=traefik-certificates-letsencrypt
# MAYAN_TRAEFIK_HTTP_PORT=80
# MAYAN_TRAEFIK_HTTPS_PORT=443
MAYAN_TRAEFIK_LETS_ENCRYPT_DNS_CHALLENGE_PROVIDER=
#########################################################
# These variables are passed to the running containers. #
......@@ -61,13 +95,9 @@ MAYAN_DOCKER_WAIT="postgresql:5432 rabbitmq:5672 redis:6379"
#########################################################
# To use block storage.
# MAYAN_PIP_INSTALLS="django-storages==1.10 boto3==1.14.58"
# MAYAN_DOCUMENTS_STORAGE_BACKEND="storages.backends.s3boto3.S3Boto3Storage"
# MAYAN_DOCUMENTS_STORAGE_BACKEND_ARGUMENTS="{'access_key':'<access key>','secret_key':'<secret key>','bucket_name':'mayan','endpoint_url':'http://<URL:port>','verify':'False'}" # 'verify':'False' for local servers with self signed SSL certificate
# To add Debian package, like LDAP.
# MAYAN_APT_INSTALLS="<Debian package names>"
# To add operating system packages, like additional OCR language,
# packages, put then in the variable below.
# MAYAN_APT_INSTALLS="tesseract-ocr-deu tesseract-ocr-nld"
......
#!make
include config.env
ifneq ($(wildcard config-local.env),)
include config-local.env
endif
HOST_IP = `/sbin/ip route|awk '/docker0/ { print $$9 }'`
APT_PROXY ?= $(HOST_IP):3142
CONSOLE_COLUMNS ?= `echo $$(tput cols)`
......@@ -14,11 +18,11 @@ DOCKER_IMAGE_MAYAN_NAME ?= $(DOCKER_IMAGE_MAYAN_NAME)
docker-build: ## Build a new image locally.
docker-build: docker-dockerfile-update
docker build --file docker/Dockerfile --tag $(DOCKER_IMAGE_MAYAN_NAME):$(IMAGE_VERSION) .
DOCKER_BUILDKIT=1 docker build --file docker/Dockerfile --tag $(DOCKER_IMAGE_MAYAN_NAME):$(IMAGE_VERSION) .
docker-build-with-proxy: ## Build a new image locally using an APT proxy as APT_PROXY.
docker-build-with-proxy: docker-dockerfile-update devpi-start
docker build --build-arg APT_PROXY=$(APT_PROXY) --build-arg PIP_INDEX_URL=$(PIP_INDEX_URL) --build-arg PIP_TRUSTED_HOST=$(PIP_TRUSTED_HOST) --build-arg HTTP_PROXY=$(HTTP_PROXY) --build-arg HTTPS_PROXY=$(HTTPS_PROXY) --file docker/Dockerfile --tag $(DOCKER_IMAGE_MAYAN_NAME):$(IMAGE_VERSION) .
DOCKER_BUILDKIT=1 docker build --build-arg APT_PROXY=$(APT_PROXY) --build-arg PIP_INDEX_URL=$(PIP_INDEX_URL) --build-arg PIP_TRUSTED_HOST=$(PIP_TRUSTED_HOST) --build-arg HTTP_PROXY=$(HTTP_PROXY) --build-arg HTTPS_PROXY=$(HTTPS_PROXY) --file docker/Dockerfile --tag $(DOCKER_IMAGE_MAYAN_NAME):$(IMAGE_VERSION) .
$(MAKE) devpi-stop
docker-build-with-proxy-push: ## Build an image with an APT proxy and push to the test registry.
......@@ -26,7 +30,7 @@ docker-build-with-proxy-push: docker-build-with-proxy docker-registry-push
docker-dockerfile-update: ## Update the Dockerfile file from the platform template.
docker-dockerfile-update: copy-config-env
./manage.py platformtemplate docker_dockerfile > docker/Dockerfile
./manage.py platform_template docker_dockerfile > docker/Dockerfile
docker-registry-push: ## Push a built image to the test Docker registry.
docker tag $(DOCKER_IMAGE_MAYAN_NAME):$(IMAGE_VERSION) $(DOCKER_HOST_REGISTRY_NAME):$(DOCKER_HOST_REGISTRY_PORT)/$(DOCKER_IMAGE_MAYAN_NAME):$(IMAGE_VERSION)
......
......@@ -6,10 +6,10 @@ x-mayan-container:
environment:
MAYAN_CELERY_BROKER_URL: amqp://${MAYAN_RABBITMQ_USER:-mayan}:${MAYAN_RABBITMQ_PASSWORD:-mayanrabbitpass}@rabbitmq:5672/${MAYAN_RABBITMQ_VHOST:-mayan}
MAYAN_CELERY_RESULT_BACKEND: redis://:${MAYAN_REDIS_PASSWORD:-mayanredispassword}@redis:6379/1
MAYAN_DATABASES: "{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'${MAYAN_DATABASE_NAME:-mayan}','PASSWORD':'${MAYAN_DATABASE_PASSWORD:-mayandbpass}','USER':'${MAYAN_DATABASE_USER:-mayan}','HOST':'${MAYAN_DATABASE_HOST:-postgresql}'}}"
MAYAN_DATABASES: "{'default':{'ENGINE':'django.db.backends.postgresql','NAME':'${MAYAN_DATABASE_NAME:-mayan}','PASSWORD':'${MAYAN_DATABASE_PASSWORD:-mayandbpass}','USER':'${MAYAN_DATABASE_USER:-mayan}','HOST':'${MAYAN_DATABASE_HOST:-postgresql}'} }"
MAYAN_LOCK_MANAGER_BACKEND: mayan.apps.lock_manager.backends.redis_lock.RedisLock
MAYAN_LOCK_MANAGER_BACKEND_ARGUMENTS: "{'redis_url':'redis://:${MAYAN_REDIS_PASSWORD:-mayanredispassword}@redis:6379/2'}"
image: ${MAYAN_DOCKER_IMAGE_NAME:-mayanedms/mayanedms}:${MAYAN_DOCKER_IMAGE_TAG:-s4.3}
image: ${MAYAN_DOCKER_IMAGE_NAME:-mayanedms/mayanedms}:${MAYAN_DOCKER_IMAGE_TAG:-s4.4}
networks:
- mayan
restart: unless-stopped
......@@ -19,7 +19,34 @@ x-mayan-container:
# - /opt/staging_folder:/staging_folder
# - /opt/watch_folder:/watch_folder
x-mayan-traefik-labels:
&mayan-traefik-labels
labels:
- "traefik.enable=${MAYAN_TRAEFIK_FRONTEND_ENABLE:-false}"
- "traefik.http.middlewares.mayan_frontend_http_redirect.redirectscheme.scheme=https"
- "traefik.http.middlewares.mayan_frontend_http_redirect.redirectscheme.permanent=false"
- "traefik.http.routers.mayan_frontend_http.entrypoints=http"
- "traefik.http.routers.mayan_frontend_http.middlewares=mayan_frontend_http_redirect"
- "traefik.http.routers.mayan_frontend_http.rule=Host(`${MAYAN_TRAEFIK_EXTERNAL_DOMAIN}`)"
- "traefik.http.routers.mayan_frontend_https.entrypoints=https"
- "traefik.http.routers.mayan_frontend_https.rule=Host(`${MAYAN_TRAEFIK_EXTERNAL_DOMAIN}`)"
- "traefik.http.routers.mayan_frontend_https.service=mayan_frontend_http"
- "traefik.http.routers.mayan_frontend_https.tls=true"
- "traefik.http.routers.mayan_frontend_https.tls.certresolver=letsencrypt"
- "traefik.http.services.mayan_frontend_http.loadbalancer.server.port=8000"
x-mayan-frontend-ports:
&mayan-frontend-ports
# Disable ports if using Traefik. Set to an empty list `[]`.
ports:
- "${MAYAN_FRONTEND_HTTP_PORT:-80}:8000"
# []
networks:
keycloak:
driver: bridge
# Change to true when using Traefik for increased security.
internal: false
mayan:
driver: bridge
# Change to true when using Traefik for increased security.
......@@ -29,11 +56,10 @@ networks:
services:
app:
<<: *mayan-container
<<: *mayan-traefik-labels
<<: *mayan-frontend-ports
profiles:
- all_in_one
# Disable ports if using Traefik.
ports:
- "80:8000"
elasticsearch:
environment:
......@@ -43,7 +69,7 @@ services:
- xpack.security.enabled=true
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- ELASTIC_PASSWORD=${MAYAN_ELASTICSEARCH_PASSWORD:-mayanespassword}
image: ${MAYAN_DOCKER_ELASTICSEARCH_IMAGE:-docker.elastic.co/elasticsearch/elasticsearch}:${MAYAN_DOCKER_ELASTICSEARCH_TAG:-7.16.0}
image: ${MAYAN_DOCKER_ELASTICSEARCH_IMAGE:-elasticsearch}:${MAYAN_DOCKER_ELASTICSEARCH_TAG:-7.17.7}
networks:
- mayan
# Enable to allow external access to the database.
......@@ -59,6 +85,60 @@ services:
volumes:
- ${MAYAN_ELASTICSEARCH_VOLUME:-elasticsearch}:/usr/share/elasticsearch/data
keycloak:
command:
- start
environment:
KEYCLOAK_ADMIN: ${MAYAN_KEYCLOAK_ADMIN:-admin}
KEYCLOAK_ADMIN_PASSWORD: ${MAYAN_KEYCLOAK_ADMIN_PASSWORD:-admin}
KC_DB: postgres
KC_DB_PASSWORD: ${MAYAN_KEYCLOAK_DATABASE_PASSWORD:-keycloakdbpass}
KC_DB_URL_DATABASE: ${MAYAN_KEYCLOAK_DATABASE_NAME:-keycloak}
KC_DB_URL_HOST: keycloak-postgres
KC_DB_USERNAME: ${MAYAN_DATABASE_KEYCLOAK_USER:-keycloak}
KC_HOSTNAME_URL: http://127.0.0.1:8081/
KC_HOSTNAME_STRICT: false
KC_HTTP_ENABLED: true
image: ${MAYAN_DOCKER_KEYCLOAK_IMAGE:-keycloak/keycloak}:${MAYAN_DOCKER_KEYCLOAK_TAG:-20.0.1}
labels:
- "traefik.enable=${MAYAN_TRAEFIK_KEYCLOAK_ENABLE:-false}"
- "traefik.http.middlewares.keycloak_http_redirect.redirectscheme.scheme=https"
- "traefik.http.middlewares.keycloak_http_redirect.redirectscheme.permanent=false"
- "traefik.http.routers.keycloak_http.entrypoints=http"
- "traefik.http.routers.keycloak_http.middlewares=keycloak_http_redirect"
- "traefik.http.routers.keycloak_http.rule=Host(`${MAYAN_TRAEFIK_EXTERNAL_DOMAIN}`)"
- "traefik.http.routers.keycloak_https.entrypoints=https"
- "traefik.http.routers.keycloak_https.rule=Host(`${MAYAN_TRAEFIK_EXTERNAL_DOMAIN}`)"
- "traefik.http.routers.keycloak_https.service=keycloak_http"
- "traefik.http.routers.keycloak_https.tls=true"
- "traefik.http.routers.keycloak_https.tls.certresolver=letsencrypt"
- "traefik.http.services.keycloak_http.loadbalancer.server.port=${MAYAN_TRAEFIK_KEYCLOAK_HTTP_PORT:-8081}"
networks:
- keycloak
- mayan
# Disable ports if using Traefik.
ports:
- "${MAYAN_TRAEFIK_KEYCLOAK_HTTP_PORT:-8081}:${MAYAN_TRAEFIK_KEYCLOAK_HTTP_PORT:-8081}"
profiles:
- keycloak
restart: unless-stopped
keycloak-postgres:
image: postgres:13.2
restart: unless-stopped
environment:
POSTGRES_DB: ${MAYAN_KEYCLOAK_DATABASE_NAME:-keycloak}
POSTGRES_PASSWORD: ${MAYAN_KEYCLOAK_DATABASE_PASSWORD:-keycloakdbpass}
POSTGRES_USER: ${MAYAN_DATABASE_KEYCLOAK_USER:-keycloak}
image: postgres:${MAYAN_DOCKER_KEYCLOAK_POSTGRES_TAG:-13.8-alpine}
networks:
- keycloak
profiles:
- keycloak_postgresql
restart: unless-stopped
volumes:
- ${MAYAN_KEYCLOAK_POSTGRES_VOLUME:-keycloak-postgres}:/var/lib/postgresql/data
postgresql:
command:
- "postgres"
......@@ -78,7 +158,7 @@ services:
POSTGRES_DB: ${MAYAN_DATABASE_NAME:-mayan}
POSTGRES_PASSWORD: ${MAYAN_DATABASE_PASSWORD:-mayandbpass}
POSTGRES_USER: ${MAYAN_DATABASE_USER:-mayan}
image: postgres:${MAYAN_DOCKER_POSTGRES_TAG:-12.10-alpine}
image: ${MAYAN_DOCKER_POSTGRES_IMAGE:-postgres}:${MAYAN_DOCKER_POSTGRES_TAG:-13.8-alpine}
networks:
- mayan
# Enable to allow external access to the database.
......@@ -109,7 +189,7 @@ services:
- "256"
- --requirepass
- "${MAYAN_REDIS_PASSWORD:-mayanredispassword}"
image: redis:${MAYAN_DOCKER_REDIS_TAG:-6.2-alpine}
image: ${MAYAN_DOCKER_REDIS_IMAGE:-redis}:${MAYAN_DOCKER_REDIS_TAG:-7.0.5-alpine}
networks:
- mayan
profiles:
......@@ -121,23 +201,10 @@ services:
# Run a frontend gunicorn container
frontend:
<<: *mayan-container
<<: *mayan-traefik-labels
<<: *mayan-frontend-ports
command:
- run_frontend
labels:
- "traefik.enable=true"
- "traefik.http.middlewares.mayan_redirect.redirectscheme.scheme=https"
- "traefik.http.middlewares.mayan_redirect.redirectscheme.permanent=false"
- "traefik.http.routers.mayan_frontend.rule=Host(`${MAYAN_TRAEFIK_EXTERNAL_DOMAIN}`)"
- "traefik.http.routers.mayan_frontend.entrypoints=http"
- "traefik.http.routers.mayan_frontend.middlewares=mayan_redirect"
- "traefik.http.routers.mayan_frontend_https.rule=Host(`${MAYAN_TRAEFIK_EXTERNAL_DOMAIN}`)"
- "traefik.http.routers.mayan_frontend_https.entrypoints=https"
- "traefik.http.routers.mayan_frontend_https.tls=true"
- "traefik.http.routers.mayan_frontend_https.tls.certresolver=letsencrypt"
- "traefik.http.services.mayan_frontend.loadbalancer.server.port=8000"
# Disable ports if using Traefik.
ports:
- "80:8000"
profiles:
- extra_frontend
......@@ -151,7 +218,7 @@ services:
entrypoint:
- /bin/sh
- -c
- 'mkdir --parents /mnt/index && chown mayan:mayan /mnt/index && /usr/local/bin/entrypoint.sh run_command "mountindex --allow-other creation_date /mnt/index"' # Replace "creation_date" with the index of your choice.
- 'mkdir --parents /mnt/index && chown mayan:mayan /mnt/index && /usr/local/bin/entrypoint.sh run_command "mirroring_mount_index --allow-other creation_date /mnt/index"' # Replace "creation_date" with the index of your choice.
profiles:
- mountindex
security_opt:
......@@ -230,21 +297,24 @@ services:
restart: "no"
rabbitmq:
image: rabbitmq:${MAYAN_DOCKER_RABBITMQ_TAG:-3.10-management-alpine}
image: ${MAYAN_DOCKER_RABBITMQ_IMAGE:-rabbitmq}:${MAYAN_DOCKER_RABBITMQ_TAG:-3.11.2-management-alpine}
environment:
RABBITMQ_DEFAULT_USER: ${MAYAN_RABBITMQ_USER:-mayan}
RABBITMQ_DEFAULT_PASS: ${MAYAN_RABBITMQ_PASSWORD:-mayanrabbitpass}
RABBITMQ_DEFAULT_VHOST: ${MAYAN_RABBITMQ_VHOST:-mayan}
labels:
- "traefik.enable=${MAYAN_TRAEFIK_RABBITMQ_ENABLE:-false}"
- "traefik.http.routers.mayan_frontend_https.tls=true"
- "traefik.http.routers.mayan_frontend.rule=Host(`${MAYAN_TRAEFIK_EXTERNAL_DOMAIN}`)"
- "traefik.http.routers.mayan_frontend.entrypoints=:15672"
- "traefik.http.routers.rabbitmq_admin_http.entrypoints=rabbitmq_admin_http"
- "traefik.http.routers.rabbitmq_admin_http.rule=Host(`${MAYAN_TRAEFIK_EXTERNAL_DOMAIN}`)"
- "traefik.http.routers.rabbitmq_admin_http.service=rabbitmq_admin_http"
- "traefik.http.routers.rabbitmq_admin_http.tls=true"
- "traefik.http.routers.rabbitmq_admin_http.tls.certresolver=letsencrypt"
- "traefik.http.services.rabbitmq_admin_http.loadbalancer.server.port=15672"
networks:
- mayan
# Enable to allow access to the administration interface.
# ports:
# - "15672:15672"
# - "${MAYAN_RABBITMQ_ADMIN_PORT:-15672}:15672"
profiles:
- rabbitmq
restart: unless-stopped
......@@ -255,33 +325,53 @@ services:
container_name: "traefik"
command:
# - "--log.level=DEBUG"
- "--api.dashboard=true"
- "--api.insecure=${MAYAN_TRAEFIK_API_INSECURE:-false}"
- "--certificatesresolvers.letsencrypt.acme.caserver=${MAYAN_TRAEFIK_LETS_ENCRYPT_SERVER:-https://acme-staging-v02.api.letsencrypt.org/directory}"
- "--certificatesresolvers.letsencrypt.acme.dnschallenge=${MAYAN_TRAEFIK_LETS_ENCRYPT_DNS_CHALLENGE:-false}"
- "--certificatesresolvers.letsencrypt.acme.dnschallenge.provider=${MAYAN_TRAEFIK_LETS_ENCRYPT_DNS_CHALLENGE_PROVIDER}"
- "--certificatesresolvers.letsencrypt.acme.email=${MAYAN_TRAEFIK_LETS_ENCRYPT_EMAIL}"
- "--certificatesresolvers.letsencrypt.acme.storage=/traefik-certificates-letsencrypt/acme.json"
- "--certificatesresolvers.letsencrypt.acme.tlschallenge=${MAYAN_TRAEFIK_LETS_ENCRYPT_TLS_CHALLENGE:-false}"
- "--entrypoints.http.address=:80"
- "--entrypoints.https.address=:443"
- "--entrypoints.keycloak_http.address=:${MAYAN_TRAEFIK_KEYCLOAK_HTTP_PORT:-8081}"
- "--entrypoints.rabbitmq_admin_http.address=:15672"
- "--entrypoints.traefik_dashboard_http.address=:8080"
- "--providers.docker=true"
- "--providers.docker.exposedbydefault=false"
image: traefik:${MAYAN_DOCKER_TRAEFIK_TAG:-v2.5}
# - Add DNS provider variables (https://doc.traefik.io/traefik/https/acme/#providers)
# environment:
image: ${MAYAN_DOCKER_TRAEFIK_IMAGE:-traefik}:${MAYAN_DOCKER_TRAEFIK_TAG:-v2.5}
labels:
- "traefik.enable=${MAYAN_TRAEFIK_DASHBOARD_ENABLE:-false}"
- "traefik.http.middlewares.basic-auth-global.basicauth.users=${MAYAN_TRAEFIK_DASHBOARD_AUTHENTICATION}"
- "traefik.http.routers.traefik_https.entrypoints=traefik_dashboard_http"
- "traefik.http.routers.traefik_https.middlewares=basic-auth-global"
- "traefik.http.routers.traefik_https.rule=Host(`${MAYAN_TRAEFIK_EXTERNAL_DOMAIN}`)"
- "traefik.http.routers.traefik_https.service=api@internal"
- "traefik.http.routers.traefik_https.tls=true"
- "traefik.http.routers.traefik_https.tls.certresolver=letsencrypt"
networks:
- mayan
- traefik
ports:
- "80:80"
- "443:443"
- "8080:8080"
- "${MAYAN_RABBITMQ_ADMIN_HTTP_PORT:-15672}:15672"
- "${MAYAN_TRAEFIK_DASHBOARD_HTTP_PORT:-8080}:8080"
- "${MAYAN_TRAEFIK_KEYCLOAK_HTTP_PORT:-8081}:8081"
- "${MAYAN_TRAEFIK_HTTP_PORT:-80}:80"
- "${MAYAN_TRAEFIK_HTTPS_PORT:-443}:443"
profiles:
- traefik
restart: unless-stopped
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
- traefik-certificates-letsencrypt:/traefik-certificates-letsencrypt
- ${MAYAN_TRAEFIK_LETSENCRYPT_VOLUME:-traefik-certificates-letsencrypt}:/traefik-certificates-letsencrypt
volumes:
app:
elasticsearch:
keycloak-postgres:
postgres:
mountindex:
rabbitmq:
......
......@@ -8,7 +8,7 @@ if [ ! "${MAYAN_QUEUE_LIST}" ]; then
echo "Must specify either MAYAN_QUEUE_LIST or MAYAN_WORKER_NAME."
exit 1
else
MAYAN_QUEUE_LIST=`su mayan --command "${MAYAN_PYTHON_BIN_DIR}mayan-edms.py platformtemplate worker_queues"`
MAYAN_QUEUE_LIST=`su mayan --command "${MAYAN_PYTHON_BIN_DIR}mayan-edms.py platform_template worker_queues"`
fi
fi
......
......@@ -58,6 +58,10 @@ X.Y # Final release
Release checklist
=================
#. Check for missing ``__init__.py`` modules::
make check-missing-inits
#. Copy the ``config.env`` values::
make copy-config-env
......@@ -122,10 +126,10 @@ Release checklist
Release using GitLab CI using the makefile
------------------------------------------
#. Use of the GitLab makefile targets: ``gitlab-release-all``,
``gitlab-release-all-minor``, ``gitlab-release-docker-minor``,
``gitlab-release-python-minor``, ``gitlab-release-docker``,
``gitlab-release-documentation``, ``gitlab-release-python``.
#. Use of the GitLab makefile targets: ``gitlab-ci-release-all``,
``gitlab-ci-release-all-minor``, ``gitlab-ci-release-docker-minor``,
``gitlab-ci-release-python-minor``, ``gitlab-ci-release-docker``,
``gitlab-ci-release-documentation``, ``gitlab-ci-release-python``.
Release using GitLab CI manually
......
.. _docker_compose:
====================
Using Docker compose
Using Docker Compose
====================
Docker Compose is the recommended way to run Mayan EDMS in production for
......
......@@ -11,38 +11,34 @@ Docker Compose installation
wget -qO- https://get.docker.com/ | sh
If you don't want to run an automated script, follow the instructions outlined
in their documentation: https://docs.docker.com/install/
If you don't want to run an automated script, follow the instructions
outlined in their documentation: https://docs.docker.com/install/
#. Install Docker Compose:
Follow the instructions as: https://docs.docker.com/compose/install/
#. Download the Mayan EDMS Docker Compose files:
For GNU/Linux based operating systems, this is usually:
The files are located at: https://gitlab.com/mayan-edms/mayan-edms/-/tree/master/docker/
.. code-block:: console
sudo curl -L "https://github.com/docker/compose/releases/download/|BINARY_DOCKER_COMPOSE_VERSION|/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
curl https://gitlab.com/mayan-edms/mayan-edms/-/raw/master/docker/docker-compose.yml -O
curl https://gitlab.com/mayan-edms/mayan-edms/-/raw/master/docker/.env -O
sudo chmod +x /usr/local/bin/docker-compose
Verify Docker Compose is working with:
#. Launch the Mayan EDMS Docker Compose containers:
.. code-block:: console
docker-compose -v
docker compose up --detach
#. Download the Mayan EDMS Docker Compose files:
The files are located at: https://gitlab.com/mayan-edms/mayan-edms/-/tree/master/docker/
.. note::
.. code-block:: console
There is no need to restart the containers after a shutdown, the
containers in the Docker Compose file are set to start automatically.
curl https://gitlab.com/mayan-edms/mayan-edms/-/raw/master/docker/docker-compose.yml -O
curl https://gitlab.com/mayan-edms/mayan-edms/-/raw/master/docker/.env -O
Customization
-------------
#. Update the Mayan EDMS Docker Compose file:
......@@ -68,60 +64,58 @@ Docker Compose installation
If the ``.env`` file does not reside in the same folder where the
``docker-compose.yml`` is or you choose another filename for it,
the option ``--env-file <you env file path>`` needs to be added
to every ``docker-compose`` invocation.
to every ``docker compose`` invocation.
#. Verify the configuration changes:
.. code-block:: console
docker-compose --file docker-compose.yml --project-name mayan config
docker compose --file docker-compose.yml --project-name mayan config
This command will output the Docker Compose file, using the values of your
``.env`` file.
#. Launch the Mayan EDMS Docker Compose containers:
.. note::
Additional commands
-------------------
If you are invoking the ``docker-compose`` command in the same directory
where the file ``docker-compose.yml`` resides, there is no need to
add the ``--file docker-compose.yml`` option.
- Invoking from and external folder:
.. code-block:: console
.. note::
docker-compose --file docker-compose.yml --project-name mayan up --detach
If you are invoking the ``docker compose`` command outside the
directory where the file ``docker-compose.yml`` resides,
you need to add the ``--file docker-compose.yml`` option.
.. note::
There is no need to restart the containers after a shutdown, the
containers in the Docker Compose file are set to start automatically.
- Multiple deployments:
Multiple deployments can be made from the same ``docker-compose.yml``
file by using the ``--project-name <project_name>`` option.
Additional commands
-------------------
- To view the logs of the containers:
.. code-block:: console
docker-compose --file docker-compose.yml --project-name mayan logs
docker compose logs
- To list the containers:
.. code-block:: console
docker-compose --file docker-compose.yml --project-name mayan ps --all
docker compose ps --all
- To terminate the containers:
.. note::
Once terminated, the containers are removed and only the data volumes remain.
Once terminated, the containers are removed and only the data volumes
remain.
.. code-block:: console
docker-compose --file docker-compose.yml --project-name mayan down
docker compose down
.. _docker_upgrading:
=========
Upgrading
=========
......
......@@ -3,33 +3,38 @@ Object Storage
==============
As introduced in :doc:`../chapters/file_storage`, Mayan EDMS takes
ownership of document backing files and stores them using the
configured storage backend. Another supported backend is the ``s3boto3``
backend. Provided by the Django Storages project, this backend
allows Mayan EDMS to use any S3 (Simple Storage Service) compatible
ownership of document files and stores them using the configured storage
backend.
Besides the default filesystem storage backend, another supported one is
the ``s3boto3`` backend provided by the Django Storages project. This
backend allows Mayan EDMS to use any S3 (Simple Storage Service) compatible
service as storage for documents, caches and tempoary files.
This backend works not only with AWS, but any other S3-compatible
API such as Minio (which can provide self-hosted S3 storage).
Using S3 storage can help Mayan EDMS scale to millions of documents.
API such as Minio. IBM Cloud Object Storage and others.
Using S3 storage it is possible to scale Mayan EDMS to millions of
documents without the need to resize the document storage volume.
The following steps will configure Mayan EDMS to use a S3 style storage
for documents on a Mayan EDMS instance
for document file storage.
.. warning::
If migrating from a different storage backend, ensure Mayan EDMS
is stopped first and that every file from the existing storage
directory is copied to the new storage location before starting
Mayan EDMS configured with with the new backend.
Also, ensure you have made a backup before beginning this process.
If migrating from a different storage backend, first make a backup
before beginning this process.
Ensure Mayan EDMS is stopped first and that every file from the
existing storage directory is copied to the new storage location with
the corresponding tool like ``s3cmd``.
.. note::
The below configuration only updates the primary
``document_storage`` location. You will need to add
additional variables if you also want to update the location
of the Caches, Temporary Location, Shared Folders, Etc.
Anything that is currently configured with
The configuration below only updates the document file storage.
You will need to add additional variables if you also want to
update the location of the Caches, Temporary location, Shared folders,
etc. Anything that is currently configured with
the ``django.core.files.storage.FileSystemStorage`` backend
can have its backend changed, but this is not advisable
for caches and temporary files locations due to performance
......@@ -44,7 +49,6 @@ in the ``environment`` section:
.. code-block:: console
MAYAN_PIP_INSTALLS: 'django-storages boto3'
MAYAN_DOCUMENTS_FILE_STORAGE_BACKEND: storages.backends.s3boto3.S3Boto3Storage
MAYAN_DOCUMENTS_FILE_STORAGE_BACKEND_ARGUMENTS: "{'access_key':'yourS3accesskey','secret_key':'yourS3secretkey','bucket_name':'yourS3bucketname','default_acl':'private'}"
......@@ -53,7 +57,6 @@ environment variable to the complete docker run command:
.. code-block:: console
-e MAYAN_PIP_INSTALLS='django-storages boto3' \
-e MAYAN_DOCUMENTS_FILE_STORAGE_BACKEND=storages.backends.s3boto3.S3Boto3Storage \
-e MAYAN_DOCUMENTS_FILE_STORAGE_BACKEND_ARGUMENTS="{'access_key':'yourS3accesskey','secret_key':'yourS3secretkey','bucket_name':'yourS3bucketname','default_acl':'private'}"
......@@ -88,15 +91,8 @@ the container if not using docker-compose.
Direct Deployments
==================
For environments installed using the direct deployment method:
Install the required dependencies:
.. code-block:: console
sudo -u mayan |MAYAN_PIP_BIN| install django-storages boto3
Then, edit the supervisor file at ``|MAYAN_SUPERVISOR_CONF|``
For environments installed using the direct deployment method, edit the
supervisor file at ``|MAYAN_SUPERVISOR_CONF|``
and add these two extra settings above the ``MAYAN_DATABASES`` option,
ensuring that the lines remain indented as before.
......@@ -149,3 +145,10 @@ You may also need to add the variables ``region_name``
and/or ``location`` depending on your S3 server configuration.
endpoint_url: "https://<my internal NAS FQDN>:9000", verify: False
Settings
========
- .. mayan_setting :: DOCUMENTS_FILE_STORAGE_BACKEND
- .. mayan_setting :: DOCUMENTS_FILE_STORAGE_BACKEND_ARGUMENTS
......@@ -42,7 +42,7 @@ Direct deployment method
- |DEFAULT_OS_DISTRIBUTION| GNU/Linux or a derivative distribution.
- Python version 3.7.
- Python version 3.9.
Recommended
......@@ -56,7 +56,7 @@ All sizes
- CPU: 64 bit, 4 cores or more, 2 GHz or faster, third generation of the
Intel Core processors (Core i7, i5, i3) "IvyBridge" or better.
- Storage: solid state drive.
- Database manager: PostgreSQL 12 or later.
- Database manager: PostgreSQL 13 or later.
Enterprise
......
......@@ -2,69 +2,450 @@
Search
======
Mayan EDMS provides a backend system that allow using different programs to
power the search function. The default backend
(``mayan.apps.dynamic_search.backends.django.DjangoSearchBackend``) uses the
database manager to perform searches.
Mayan EDMS search system works by providing backends for different search
engines. Backend classes acts like brides between the specialized search
engine functionality and the uniform code in Mayan EDMS. Query types,
operators and data types are interpreted before passing them to the search
engine to provide a similar experience regardless of the search engine.
Search syntax
=============
Search syntax support will depend on the search backend being used. The
``DjangoSearchBackend`` backend supports the following syntax.
Text entered in the search bar are split into search terms. The blank space
character is used as the term delimiter.
Search types
------------
Three search types are provided by default. These are simple, advanced,
and scoped.
When performing a simple search, only the search terms are required. These
terms are then matched against all the fields.
The advanced search provides more control by allowing you to specify which
terms are to be matched against which fields.
The scoped search provides the most control by allowing you to perform
logical grouping of search term to field matches.
Terms
-----
Text used for search is split into search terms. The blank space
character is used as the term delimiter. To use a phrase as the search term,
enclose it using double quotes.
Operators
---------
Operators allow combining the results of search terms. Three operators
are supported: AND, OR, NOT.
By default, search terms are routed to an ``AND`` query. That means
that a search for:
The following search terms:
``Tag1 Tag2``
``Tag1 AND Tag2``
will only return documents with both tags attached. To offer the
opposite choice we added an ``OR`` syntax. Searching for:
will only return the results that match two terms.
These next search terms:
``Tag1 OR Tag2``
will return documents with either tag attached.
will return results matching either term.
Literals terms are also supported.
Search exclusions are also supported via the ``NOT`` operator.
Searching for:
``Tag1 NOT Tag2``
will return results matching ``Tag1`` and not matching ``Tag2``.
Operators are processed in the same order as they are entered.
If no operator is specified, the fallback operator will be used. By default
this is the ``AND`` operator. This can be changed via the setting named
:ref:`settings-SEARCH-DEFAULT-OPERATOR`.
Quoted terms
------------
Entering the search terms:
``blue car``
will return documents with the words ``blue`` and ``car``, even if they are
not together. That means getting documents with the phrases ``blue sky``
and ``slow car``. To search for exact terms enclose them in quotes:
will return results with the words ``blue`` and ``car``, even if they are
not together or in a different order. That means getting results with the
phrase ``blue sky`` and ``slow car``. To handle both words as a single term
enclose them in quotes:
``"blue car"``
This will return only documents with the exact phrase "blue car".
This will return only results with the phrase "blue car".
Query types
-----------
.. versionadded:: 4.4
Query types allow matching the search terms to the fields in different ways.
The following query types are supported by the search preprocessor. These
query types are then converted to the backend search engine specific queries.
Not all query types might be supported by a specific search engine.
Exact
^^^^^
Match the exact term.
Alias: ``=``
Example: ``=blue``
Fuzzy
^^^^^
Match the variations of the term where a letter might be transposed.
Alias: ``~``
Example: ``~charactre``. This will match ``character``.
Greater than
^^^^^^^^^^^^
Matches values greater than the term. Works for numbers, text, and dates.
Alias: ``>``
Example: ``>2022-10-15``
Greater than or equal
^^^^^^^^^^^^^^^^^^^^^
Matches values greater than or equal to the term. Works for numbers, text,
and dates.
Alias: ``>=``
Example: ``>50``
Less than
^^^^^^^^^
Matches values less than to the term. Works for numbers, text, and dates.
Alias: ``<``
Example: ``<2023-01``
Less than or equal
^^^^^^^^^^^^^^^^^^
Matches values greater than or equal to the term. Works for numbers, text,
and dates.
Alias: ``<=``
Example: ``<=100``
Partial
^^^^^^^
Matches the term as if it is a fragment.
This is the default query type if no query type is specified.
Alias: ``*``
Example: ``*play``. This will match ``play``, ``played``, ``plays``,
``playing``.
Range (inclusive)
^^^^^^^^^^^^^^^^^
Matches values between the terms specified in the range, including the terms
themselves. Works for numbers and dates.
Alias: ``[]``
Example: ``[]1..100``. This will match all numbers between 1 and 100.
Range (exclusive)
^^^^^^^^^^^^^^^^^
Matches values between the terms specified in the range, not including the
terms themselves. Works for numbers and dates.
Alias: ``{}``
Example: ``{}1..100``. This will match all numbers greater than 1 and less
than 100.
Regular expression
^^^^^^^^^^^^^^^^^^
Performs a match using a regular expression. The regular expression is passed
directly to the search engine. Each search engine supports a different
regular expression dialect. Consult the documentation of the search engine to
learn more about their regular expression support and syntax.
Alias: ``%``
Example: ``%^abc[1-4]``. This will match all entries that start with ``abc``
and are followed by a digit from 1 to 4.
Data types
----------
.. versionadded:: 4.4
The preprocessor will parse terms to a common data type before converting
it further to the specific data type required by the search engine. This
bridges the gap regarding which data types are support and homogenizes the
format of each data type.
Negative terms or exclusions is also support by adding a minus (``-``)
character to the term to be excluded.
Integers
^^^^^^^^
The search terms:
Example: ``1``, ``150``
``Tag1 -Tag2``
will return documents that have the ``Tag1`` attached and that do not have
``Tag2`` attached.
Boolean
^^^^^^^
Boolean values can be either the words ``true`` or ``false``. Casing is
ignored.
Other backends
Example: ``true``, ``FALSE``, ``True``
Date and time
^^^^^^^^^^^^^
To specify a date at least the year must be entered. Every other date and
time element is optional. If the month is not specified it defaults to
January. If the day is not specified it defaults to 1. If the hour, minute
or second are not specified they each default to 0.
Besides dates in electronic format, this data type also supports dates in
human format.
Example: ``2022``, ``2022-6``, ``2022-03-11T10:11:10``
The following humanized dates are also supported:
- ``today``
- ``this month``
- ``this year``
- ``<number> month ago``
Dates can be used in range queries:
- ``"[]one month ago..today"``
- ``[]2000-01-01..2020-12-31``
- ``[]2020-01-01T1-10..today"``
Raw terms
---------
.. versionadded:: 4.4
Raw terms also useful when you want to access specific search engine
features. Raw terms are passed as-is to the search engine. Enclose the term
or phrase inside backticks to mark it as a raw term.
In the search term:
```blue AND car```
The ``AND`` operator is not processed by Mayan EDMS. Instead the whole phrase
is passed to the search engine. In this case, it will be up to the search
engine how the operator (or any other syntax) is processed.
Data normalization
==================
.. versionadded:: 4.4
Data and user queries are normalized using a pipeline of value
transformations. These value transformations vary by data type and by
search backend.
Text
----
Text is normalized using case folding which not only converts text to
lowercase but also converts language specific letter to other aliases.
For example the German letter ``ß`` is converted to ``ss``. Since this
conversion happens during indexing and searching, this serves to provide
search engine independent case insensitive searches.
Hyphens are also normalized. The hyphen symbol usually has special
meaning to search engines. To avoid contextual collisions, hyphens
are converted to underscores. This causes all hyphenated words to
be treated as a single word. Hyphens during search are also converted
to underscores normalizing the search query with the indexed content.
Accented words are normalized to their unaccented form.
Dates
-----
Dates are normalized to the date specific format required by the
search engine. This could be a Python datetime object, an
ISO formatted date or a search engine specific date format.
For maximum compatibility the microseconds part of the date time value is
removed.
Numbers
-------
Numbers are normalized to integers. Signs are not removed.
UUID
----
UUID fields are normalized to text. The hyphen is stripped and the UUID is
indexed as a single string.
Empty values
============
To search for empty values use an empty quoted term.
Example: ``""``
Backends
========
Django
------
Path: ``mayan.apps.dynamic_search.backends.django.DjangoSearchBackend``
This was the first backend supported. It uses the same database as the
rest of the system to emulate a search engine.
As it uses the database, external services or reindexing to update its
content is not required.
The downside to this backend is that it is slow and can overload the database
affecting the entire performance of the deployment.
Since version 4.2 it is no longer the default search backend.
Unsupported features
^^^^^^^^^^^^^^^^^^^^
- Accent folding.
- Case folding.
- Fuzzy searches are emulated and might not return the same results as
a search engine that has native support for fuzzy searches.
Whoosh
------
.. versionadded:: 3.5
Path: ``mayan.apps.dynamic_search.backends.whoosh.WhooshSearchBackend``
This backend uses the Python Whoosh search library. Whoosh uses local files
for indexing. Because of this, it runs in the same context as Mayan EDMS,
no external services are required. Using and backing up Whoosh is very easy.
The downside to this backend is that it can only be used when Mayan EDMS is
configure to use block storage. Mayan EDMS implementation of Whoosh also
uses a distributed lock to avoid concurrent writing and possible corruption.
This slows down the update process of the search index.
This backend provides search functionality that is simple to setup and
will work well from small to intermediate installations.
In version 4.2, the Whoosh backend was completed and became the default
search backend.
This engine support specialized date parsing. To use this feature, pass the
date term as a raw term.
Example: ``=`['last tuesday' to 'next friday']```
More examples of date parsing can be found in
https://whoosh.readthedocs.io/en/latest/dates.html#parsing-date-queries
To pass reserved characters or symbols that have special meaning to the
preprocessor or to the Whoosh parsers, pass them as a raw term and
also enclose them in single quotes.
Example: To search for the terms with the ``<`` symbol use ``=`'<'```
More details can be found in
https://whoosh.readthedocs.io/en/latest/querylang.html#making-a-term-from-literal-text
ElasticSearch
-------------
.. versionadded:: 4.2
Path: ``mayan.apps.dynamic_search.backends.elasticsearch.ElasticSearchBackend``
This backend uses ElasticSearch via a local API client. ElasticSearch must
be deployed as an external service, either manually or automatically using
the official Docker Compose file.
ElasticSearch can scale up very well and support millions of documents and
many concurrent search requests. ElasticSearch can also be clustered to
add more capabilities.
The downside is that ElasicSearch has high resource requirements and has
an extensive but complex search syntax. Mayan EDMS only uses a subset of the
search features provided by ElasticSearch.
This backend is recommended for large installations having a high number of
documents and concurrent users.
Considerations
==============
Besides the default Django search backend, a backend using the Whoosh search
engine is included. The name of this backend is
``mayan.apps.dynamic_search.backends.whoosh.WhooshSearchBackend`` and can be
enabled by changing the value of the setting ``SEARCH_BACKEND``. When changing
the search backend, it is also necessary to launch the "Reindex search backend"
action from the Tools menu to initialize the search engine index. Indexing the
search engine is only required once, afterwards the search engine will be
updated as documents are added or edited.
When changing the search backend, it is also necessary to launch the
"Reindex search backend" action from the Tools menu to initialize the
search engine index.
This action is only required once, afterwards the search engine will be
updated as objects are added, removed, or edited.
Settings
========
Refer to Whoosh's documentation for more information about the search syntax
it supports: https://whoosh.readthedocs.io/en/latest/querylang.html
.. mayan_setting_namespace :: search
......@@ -14,17 +14,19 @@
import os
import sys
from docutils.parsers.rst import directives
import docutils.parsers.rst
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mayan.settings')
sys.path.insert(0, os.path.abspath('..'))
sys.path.insert(1, os.path.abspath('.'))
from contrib.scripts.version import Version
import django
import mayan
import callbacks
import directives
import patches
import utils
......@@ -274,6 +276,9 @@ html_baseurl = 'https://docs.mayan-edms.com/'
def setup(app):
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'mayan.settings')
django.setup()
environment_variables = utils.load_env_file()
MAYAN_PYTHON_BIN_DIR = os.path.join(
......@@ -313,7 +318,15 @@ def setup(app):
substitutions=substitutions
)
)
directives.register_directive(
app.add_directive(
name='mayan_setting_namespace',
cls=directives.DirectiveMayanSettingNamespace
)
app.add_directive(
name='mayan_setting',
cls=directives.DirectiveMayanSetting
)
docutils.parsers.rst.directives.register_directive(
name='include', directive=patches.monkey_patch_include(
substitutions=substitutions
)
......