Skip to content

With mattermost enabled, service restart (chef recipe) will fail

Summary

With mattermost enabled, service restart will fail.

Steps to reproduce

  • start with a docker-compose.yml file like this:
services:
  gitlab:
    image: gitlab/gitlab-ce
    environment:
      GITLAB_OMNIBUS_CONFIG: |
        mattermost_external_url 'https://mattermost.my-example.com'
...

  • run docker-compose up -d to start the service (with mattermost enabled)

  • run docker-compose restart gitlab will reproduce the error (see "Relevant logs" below) and the docker container exits

What is the current bug behavior?

Docker container exits after restart.

What is the expected correct behavior?

Docker container restarts successfully.

Relevant logs

Relevant logs 
...
Recipe: mattermost::enable
  * service[mattermost] action restart



================================================================================
Error executing action `restart` on resource 'service[mattermost]'
================================================================================

Mixlib::ShellOut::ShellCommandFailed
------------------------------------
Expected process to exit with [0], but received '1'
---- Begin output of /opt/gitlab/embedded/bin/chpst -u root /opt/gitlab/embedded/bin/sv restart /opt/gitlab/service/mattermost ----
STDOUT: fail: /opt/gitlab/service/mattermost: runsv not running
STDERR:
---- End output of /opt/gitlab/embedded/bin/chpst -u root /opt/gitlab/embedded/bin/sv restart /opt/gitlab/service/mattermost ----
Ran /opt/gitlab/embedded/bin/chpst -u root /opt/gitlab/embedded/bin/sv restart /opt/gitlab/service/mattermost returned 1

Resource Declaration:
---------------------
# In /opt/gitlab/embedded/cookbooks/cache/cookbooks/runit/definitions/runit_service.rb

227:     service params[:name] do
228:       control_cmd = node[:runit][:sv_bin]
229:       if params[:owner]
230:         control_cmd = "#{node[:runit][:chpst_bin]} -u #{params[:owner]} #{control_cmd}"
231:       end
232:       provider Chef::Provider::Service::Simple

Compiled Resource:
------------------
# Declared in /opt/gitlab/embedded/cookbooks/cache/cookbooks/runit/definitions/runit_service.rb:227:in `block in from_file'

service("mattermost") do
  params {:directory=>"/opt/gitlab/sv", :only_if=>false, :finish_script=>false, :control=>[], :run_restart=>true, :active_directory=>"/opt/gitlab/service", :init_script_template=>nil, :owner=>"root", :group=>"root", :template_name=>"mattermost", :start_command=>"start", :stop_command=>"stop", :restart_command=>"restart", :status_command=>"status", :options=>{:log_directory=>"/var/log/gitlab/mattermost/", :directory=>nil, :only_if=>false, :finish_script=>false, :control=>[], :run_restart=>true, :active_directory=>nil, :init_script_template=>nil, :owner=>"root", :group=>"root", :template_name=>nil, :start_command=>"start", :stop_command=>"stop", :restart_command=>"restart", :status_command=>"status", :options=>{}, :log_options=>{}, :env=>{}, :action=>:enable, :down=>false, :supervisor_owner=>nil, :supervisor_group=>nil}, :log_options=>{"svlogd_size"=>209715200, "svlogd_num"=>30, "svlogd_timeout"=>86400, "svlogd_filter"=>"gzip", "svlogd_udp"=>nil, "svlogd_prefix"=>nil, "udp_log_shipping_host"=>nil, "udp_log_shipping_hostname"=>nil, "udp_log_shipping_port"=>514, "logrotate_frequency"=>"daily", "logrotate_size"=>nil, "logrotate_rotate"=>30, "logrotate_compress"=>"compress", "logrotate_method"=>"copytruncate", "logrotate_postrotate"=>nil, "logrotate_dateformat"=>nil, "enable"=>true, "username"=>"mattermost", "group"=>"mattermost", "uid"=>nil, "gid"=>nil, "home"=>"/var/opt/gitlab/mattermost", "database_name"=>"mattermost_production", "env"=>{"MM_SERVICESETTINGS_ENABLEINCOMINGWEBHOOKS"=>"true", "MM_SERVICESETTINGS_ENABLEOUTGOINGWEBHOOKS"=>"true", "MM_SERVICESETTINGS_ENABLELINKPREVIEWS"=>"true", "MM_EMAILSETTINGS_ENABLESIGNINWITHEMAIL"=>"false", "MM_EMAILSETTINGS_SENDEMAILNOTIFICATIONS"=>"true", "MM_EMAILSETTINGS_FEEDBACKNAME"=>"Mattermost", "MM_EMAILSETTINGS_FEEDBACKEMAIL"=>"noreply@my-example.com", "MM_EMAILSETTINGS_ENABLESMTPAUTH"=>"true", "MM_EMAILSETTINGS_SMTPUSERNAME"=>"*****", "MM_EMAILSETTINGS_SMTPPASSWORD"=>"*****", "MM_EMAILSETTINGS_SMTPSERVER"=>"*****", "MM_EMAILSETTINGS_SMTPPORT"=>"25", "MM_EMAILSETTINGS_CONNECTIONSECURITY"=>"STARTTLS", "MM_EMAILSETTINGS_ENABLEEMAILBATCHING"=>"true", "MM_EMAILSETTINGS_SKIPSERVERCERTIFICATEVERIFICATION"=>"true", "MM_EMAILSETTINGS_SENDPUSHNOTIFICATIONS"=>"true", "MM_EMAILSETTINGS_PUSHNOTIFICATIONSERVER"=>"http://push.mattermost.com"}, "log_file_directory"=>"/var/log/gitlab/mattermost/", "log_console_enable"=>true, "log_enable_webhook_debugging"=>true, "log_console_level"=>"INFO", "log_enable_file"=>true, "log_file_level"=>"ERROR", "log_file_format"=>nil, "log_enable_diagnostics"=>true, "service_use_ssl"=>true, "service_address"=>"127.0.0.1", "service_port"=>"8065", "service_site_url"=>"https://mattermost.my-example.com", "service_websocket_url"=>nil, "service_maximum_login_attempts"=>10, "service_google_developer_key"=>nil, "service_enable_incoming_webhooks"=>false, "service_enable_post_username_override"=>true, "service_enable_post_icon_override"=>true, "service_enable_testing"=>false, "service_enable_security_fix_alert"=>true, "service_enable_insecure_outgoing_connections"=>false, "service_allowed_untrusted_internal_connections"=>" gitlab.my-example.com", "service_allow_cors_from"=>nil, "service_allow_cookies_from_subdomains"=>false, "service_enable_outgoing_webhooks"=>false, "service_enable_commands"=>true, "service_enable_custom_emoji"=>false, "service_enable_oauth_service_provider"=>false, "service_enable_developer"=>false, "service_session_length_web_in_days"=>30, "service_session_length_mobile_in_days"=>30, "service_session_length_sso_in_days"=>30, "service_session_cache_in_minutes"=>10, "service_session_idle_timeout_in_minutes"=>0, "service_websocket_secure_port"=>443, "service_websocket_port"=>80, "service_webserver_mode"=>"gzip", "service_connection_security"=>nil, "service_tls_cert_file"=>nil, "service_tls_key_file"=>nil, "service_use_lets_encrypt"=>false, "service_lets_encrypt_cert_cache_file"=>"./config/letsencrypt.cache", "service_forward_80_to_443"=>false, "service_read_timeout"=>300, "service_write_timeout"=>300, "service_time_between_user_typing_updates_milliseconds"=>5000, "service_enable_link_previews"=>false, "service_enable_user_typing_messages"=>true, "service_enable_post_search"=>true, "service_enable_user_statuses"=>true, "service_enable_emoji_picker"=>true, "service_enable_channel_viewed_messages"=>true, "service_enable_apiv3"=>true, "service_goroutine_health_threshold"=>-1, "service_user_access_tokens"=>false, "service_enable_preview_features"=>true, "service_close_unused_direct_messages"=>false, "service_image_proxy_type"=>"", "service_image_proxy_url"=>"", "sql_driver_name"=>"postgres", "sql_data_source"=>"user=gitlab_mattermost host=/var/opt/gitlab/postgresql port=5432 dbname=mattermost_production", "sql_data_source_replicas"=>["user=gitlab_mattermost host=/var/opt/gitlab/postgresql port=5432 dbname=mattermost_production"], "sql_max_idle_conns"=>10, "sql_max_open_conns"=>10, "sql_trace"=>false, "sql_data_source_search_replicas"=>[], "sql_query_timeout"=>30, "gitlab"=>{}, "file_max_file_size"=>52428800, "file_driver_name"=>"local", "file_directory"=>"/var/opt/gitlab/mattermost/data", "file_enable_public_link"=>true, "file_initial_font"=>"luximbi.ttf", "file_amazon_s3_access_key_id"=>nil, "file_amazon_s3_bucket"=>nil, "file_amazon_s3_secret_access_key"=>nil, "file_amazon_s3_endpoint"=>nil, "file_amazon_s3_ssl"=>true, "file_amazon_s3_sign_v2"=>false, "file_enable_file_attachments"=>true, "file_amazon_s3_trace"=>false, "email_enable_sign_up_with_email"=>false, "email_enable_sign_in_with_email"=>true, "email_enable_sign_in_with_username"=>false, "email_send_email_notifications"=>false, "email_use_channel_in_email_notifications"=>true, "email_require_email_verification"=>false, "email_feedback_name"=>nil, "email_feedback_email"=>nil, "email_feedback_organization"=>nil, "email_smtp_username"=>nil, "email_smtp_password"=>nil, "email_smtp_server"=>nil, "email_smtp_port"=>nil, "email_connection_security"=>nil, "email_send_push_notifications"=>false, "email_push_notification_server"=>nil, "email_push_notification_contents"=>"generic", "email_enable_batching"=>false, "email_batching_buffer_size"=>256, "email_batching_interval"=>30, "email_skip_server_certificate_verification"=>false, "email_smtp_auth"=>false, "email_notification_content_type"=>"full", "ratelimit_enable_rate_limiter"=>false, "ratelimit_per_sec"=>10, "ratelimit_max_burst"=>100, "ratelimit_memory_store_size"=>10000, "ratelimit_vary_by_remote_addr"=>true, "ratelimit_vary_by_user"=>false, "ratelimit_vary_by_header"=>nil, "privacy_show_email_address"=>true, "privacy_show_full_name"=>true, "localization_server_locale"=>"en", "localization_client_locale"=>"en", "localization_available_locales"=>"", "team_site_name"=>"GitLab Mattermost", "team_enable_user_creation"=>true, "team_enable_open_server"=>false, "team_enable_x_to_leave_channels_from_lhs"=>false, "team_max_users_per_team"=>150, "team_allow_public_link"=>true, "team_allow_valet_default"=>false, "team_restrict_creation_to_domains"=>nil, "team_restrict_direct_message"=>"any", "team_max_channels_per_team"=>2000, "team_max_notifications_per_channel"=>1000, "team_user_status_away_timeout"=>300, "team_enable_confirm_notifications_to_channel"=>true, "team_teammate_name_display"=>"full_name", "support_terms_of_service_link"=>"https://about.mattermost.com/default-terms/", "support_privacy_policy_link"=>"https://about.mattermost.com/default-privacy-policy/", "support_about_link"=>"https://about.mattermost.com/default-about/", "support_help_link"=>"https://about.mattermost.com/default-help/", "support_report_a_problem_link"=>"https://about.mattermost.com/default-report-a-problem/", "support_email"=>"support@example.com", "gitlab_enable"=>true, "gitlab_secret"=>"*****", "gitlab_id"=>"*****", "gitlab_scope"=>"", "gitlab_auth_endpoint"=>"https://gitlab.my-example.com/oauth/authorize", "gitlab_token_endpoint"=>"https://gitlab.my-example.com/oauth/token", "gitlab_user_api_endpoint"=>"https://gitlab.my-example.com/api/v4/user", "webrtc_enable"=>false, "webrtc_gateway_websocket_url"=>nil, "webrtc_gateway_admin_url"=>nil, "webrtc_gateway_admin_secret"=>nil, "webrtc_gateway_stun_uri"=>nil, "webrtc_gateway_turn_uri"=>nil, "webrtc_gateway_turn_username"=>nil, "webrtc_gateway_turn_shared_key"=>nil, "native_app_app_download_link"=>"https://about.mattermost.com/downloads/", "native_app_android_app_download_link"=>"https://about.mattermost.com/mattermost-android-app/", "native_app_ios_app_download_link"=>"https://about.mattermost.com/mattermost-ios-app/", "job_run_jobs"=>true, "job_run_scheduler"=>true, "plugin_enable"=>true, "plugin_enable_uploads"=>false, "plugin_directory"=>"/var/opt/gitlab/mattermost/plugins", "plugin_client_directory"=>"/var/opt/gitlab/mattermost/client-plugins", "plugin_plugins"=>{}, "plugin_plugin_states"=>{}, "email_invite_salt"=>"*****", "file_public_link_salt"=>"287c67d4e2bc04732a14dbc1c000a046", "sql_at_rest_encrypt_key"=>"555ace159de1ba2accfcf94ee6fde904", "host"=>"mattermost.my-example.com", "port"=>443}, :env=>{}, :action=>:enable, :down=>false, :supervisor_owner=>nil, :supervisor_group=>nil, :name=>"mattermost"}
  provider Chef::Provider::Service::Simple
  action [:nothing]
  default_guard_interpreter :default
  service_name "mattermost"
  enabled nil
  running nil
  masked nil
  pattern "mattermost"
  start_command "/opt/gitlab/embedded/bin/chpst -u root /opt/gitlab/embedded/bin/sv start /opt/gitlab/service/mattermost"
  stop_command "/opt/gitlab/embedded/bin/chpst -u root /opt/gitlab/embedded/bin/sv stop /opt/gitlab/service/mattermost"
  status_command "/opt/gitlab/embedded/bin/chpst -u root /opt/gitlab/embedded/bin/sv status /opt/gitlab/service/mattermost"
  restart_command "/opt/gitlab/embedded/bin/chpst -u root /opt/gitlab/embedded/bin/sv restart /opt/gitlab/service/mattermost"
  declared_type :service
  cookbook_name "mattermost"
  recipe_name "enable"
  supports {:restart=>true, :status=>true}
end

System Info:
------------
chef_version=13.6.4
platform=ubuntu
platform_version=16.04
ruby=ruby 2.3.7p456 (2018-03-28 revision 63024) [x86_64-linux]
program_name=/opt/gitlab/embedded/bin/chef-client
executable=/opt/gitlab/embedded/bin/chef-client





Running handlers:
Running handlers complete
There was an error running gitlab-ctl reconfigure:


service[mattermost] (mattermost::enable line 227) had an error: Mixlib::ShellOut::ShellCommandFailed: Expected process to exit with [0], but received '1'
---- Begin output of /opt/gitlab/embedded/bin/chpst -u root /opt/gitlab/embedded/bin/sv restart /opt/gitlab/service/mattermost ----
STDOUT: fail: /opt/gitlab/service/mattermost: runsv not running
STDERR:
---- End output of /opt/gitlab/embedded/bin/chpst -u root /opt/gitlab/embedded/bin/sv restart /opt/gitlab/service/mattermost ----
Ran /opt/gitlab/embedded/bin/chpst -u root /opt/gitlab/embedded/bin/sv restart /opt/gitlab/service/mattermost returned 1


Chef Client failed. 19 resources updated in 10 seconds
...

Details of package version

Provide the package version installation details 
docker image tag:  
gitlab/gitlab-ce:10.8.3-ce.0

gitlab version:

gitlab 10.8.3 (564c342)

Environment details

  • Operating System: CentOS Linux release 7.5.1804 (Core)
  • Installation Target, remove incorrect values:
    • VM: Digital Ocean
  • Installation Type, remove incorrect values:
    • Other: docker with version 18.03.1-ce
  • Is there any other software running on the machine: docker-ce
  • Is this a single or multiple node installation?
  • Resources
    • CPU: 2-core, 4-thread
    • Memory total: 7.6G

Configuration details

Provide the relevant sections of `/etc/gitlab/gitlab.rb` 
docker omnibus default
(empty in /etc/gitlab/gitlab.rb)