From: Tom Hughes Date: Thu, 23 Jul 2020 18:03:43 +0000 (+0100) Subject: Standardise generation of FPM pool configurations X-Git-Url: https://git.openstreetmap.org./chef.git/commitdiff_plain/867a767ee96c65b89179c9d53f8c7805173f3fa3 Standardise generation of FPM pool configurations --- diff --git a/cookbooks/dev/recipes/default.rb b/cookbooks/dev/recipes/default.rb index be4e96c68..30d26f910 100644 --- a/cookbooks/dev/recipes/default.rb +++ b/cookbooks/dev/recipes/default.rb @@ -94,7 +94,11 @@ end package "apache2-suexec-pristine" php_fpm "default" do - template "fpm-default.conf.erb" + port 7000 + pm_max_children 10 + pm_start_servers 4 + pm_min_spare_servers 2 + pm_max_spare_servers 6 end php_fpm "www" do @@ -155,11 +159,30 @@ search(:accounts, "*:*").each do |account| next unless File.directory?("#{user_home}/public_html") - port = 7000 + account["uid"].to_i + port_number = 7000 + account["uid"].to_i php_fpm name do - template "fpm.conf.erb" - variables :user => name, :port => port + port port_number + user name + group name + pm_max_children 10 + pm_start_servers 4 + pm_min_spare_servers 2 + pm_max_spare_servers 6 + pm_max_requests 10000 + request_terminate_timeout 1800 + environment "HOSTNAME" => "$HOSTNAME", + "PATH" => "/usr/local/bin:/usr/bin:/bin", + "TMP" => "/tmp", + "TMPDIR" => "/tmp", + "TEMP" => "/tmp" + php_values "max_execution_time" => "300", + "memory_limit" => "128M", + "post_max_size" => "32M", + "upload_max_filesize" => "32M" + php_admin_values "sendmail_path" => "/usr/sbin/sendmail -t -i -f #{name}@errol.openstreetmap.org", + "open_basedir" => "/home/#{name}/:/tmp/:/usr/share/php/" + php_flags "display_errors" => "on" end ssl_certificate "#{name}.dev.openstreetmap.org" do @@ -170,7 +193,7 @@ search(:accounts, "*:*").each do |account| apache_site "#{name}.dev.openstreetmap.org" do template "apache.user.erb" directory "#{user_home}/public_html" - variables :user => name, :port => port + variables :user => name, :port => port_number end template "/etc/sudoers.d/#{name}" do diff --git a/cookbooks/dev/templates/default/fpm-default.conf.erb b/cookbooks/dev/templates/default/fpm-default.conf.erb deleted file mode 100644 index fe7ee4c40..000000000 --- a/cookbooks/dev/templates/default/fpm-default.conf.erb +++ /dev/null @@ -1,384 +0,0 @@ -; Start a new pool named 'default'. -; the variable $pool can we used in any directive and will be replaced by the -; pool name ('default' here) -[default] - -; Per pool prefix -; It only applies on the following directives: -; - 'slowlog' -; - 'listen' (unixsocket) -; - 'chroot' -; - 'chdir' -; - 'php_values' -; - 'php_admin_values' -; When not set, the global prefix (or /usr) applies instead. -; Note: This directive can also be relative to the global prefix. -; Default Value: none -;prefix = /path/to/pools/$pool - -; Unix user/group of processes -; Note: The user is mandatory. If the group is not set, the default user's group -; will be used. -user = www-data -group = www-data - -; The address on which to accept FastCGI requests. -; Valid syntaxes are: -; 'ip.add.re.ss:port' - to listen on a TCP socket to a specific address on -; a specific port; -; 'port' - to listen on a TCP socket to all addresses on a -; specific port; -; '/path/to/unix/socket' - to listen on a unix socket. -; Note: This value is mandatory. -listen = 127.0.0.1:7000 - -; Set listen(2) backlog. A value of '-1' means unlimited. -; Default Value: 128 (-1 on FreeBSD and OpenBSD) -;listen.backlog = -1 - -; Set permissions for unix socket, if one is used. In Linux, read/write -; permissions must be set in order to allow connections from a web server. Many -; BSD-derived systems allow connections regardless of permissions. -; Default Values: user and group are set as the running user -; mode is set to 0666 -;listen.owner = www-data -;listen.group = www-data -;listen.mode = 0660 - -; List of ipv4 addresses of FastCGI clients which are allowed to connect. -; Equivalent to the FCGI_WEB_SERVER_ADDRS environment variable in the original -; PHP FCGI (5.2.2+). Makes sense only with a tcp listening socket. Each address -; must be separated by a comma. If this value is left blank, connections will be -; accepted from any ip address. -; Default Value: any -;listen.allowed_clients = 127.0.0.1 - -; Choose how the process manager will control the number of child processes. -; Possible Values: -; static - a fixed number (pm.max_children) of child processes; -; dynamic - the number of child processes are set dynamically based on the -; following directives. With this process management, there will be -; always at least 1 children. -; pm.max_children - the maximum number of children that can -; be alive at the same time. -; pm.start_servers - the number of children created on startup. -; pm.min_spare_servers - the minimum number of children in 'idle' -; state (waiting to process). If the number -; of 'idle' processes is less than this -; number then some children will be created. -; pm.max_spare_servers - the maximum number of children in 'idle' -; state (waiting to process). If the number -; of 'idle' processes is greater than this -; number then some children will be killed. -; ondemand - no children are created at startup. Children will be forked when -; new requests will connect. The following parameter are used: -; pm.max_children - the maximum number of children that -; can be alive at the same time. -; pm.process_idle_timeout - The number of seconds after which -; an idle process will be killed. -; Note: This value is mandatory. -pm = dynamic - -; The number of child processes to be created when pm is set to 'static' and the -; maximum number of child processes when pm is set to 'dynamic' or 'ondemand'. -; This value sets the limit on the number of simultaneous requests that will be -; served. Equivalent to the ApacheMaxClients directive with mpm_prefork. -; Equivalent to the PHP_FCGI_CHILDREN environment variable in the original PHP -; CGI. The below defaults are based on a server without much resources. Don't -; forget to tweak pm.* to fit your needs. -; Note: Used when pm is set to 'static', 'dynamic' or 'ondemand' -; Note: This value is mandatory. -pm.max_children = 10 - -; The number of child processes created on startup. -; Note: Used only when pm is set to 'dynamic' -; Default Value: min_spare_servers + (max_spare_servers - min_spare_servers) / 2 -pm.start_servers = 4 - -; The desired minimum number of idle server processes. -; Note: Used only when pm is set to 'dynamic' -; Note: Mandatory when pm is set to 'dynamic' -pm.min_spare_servers = 2 - -; The desired maximum number of idle server processes. -; Note: Used only when pm is set to 'dynamic' -; Note: Mandatory when pm is set to 'dynamic' -pm.max_spare_servers = 6 - -; The number of seconds after which an idle process will be killed. -; Note: Used only when pm is set to 'ondemand' -; Default Value: 10s -;pm.process_idle_timeout = 10s; - -; The number of requests each child process should execute before respawning. -; This can be useful to work around memory leaks in 3rd party libraries. For -; endless request processing specify '0'. Equivalent to PHP_FCGI_MAX_REQUESTS. -; Default Value: 0 -;pm.max_requests = 500 - -; The URI to view the FPM status page. If this value is not set, no URI will be -; recognized as a status page. It shows the following informations: -; pool - the name of the pool; -; process manager - static, dynamic or ondemand; -; start time - the date and time FPM has started; -; start since - number of seconds since FPM has started; -; accepted conn - the number of request accepted by the pool; -; listen queue - the number of request in the queue of pending -; connections (see backlog in listen(2)); -; max listen queue - the maximum number of requests in the queue -; of pending connections since FPM has started; -; listen queue len - the size of the socket queue of pending connections; -; idle processes - the number of idle processes; -; active processes - the number of active processes; -; total processes - the number of idle + active processes; -; max active processes - the maximum number of active processes since FPM -; has started; -; max children reached - number of times, the process limit has been reached, -; when pm tries to start more children (works only for -; pm 'dynamic' and 'ondemand'); -; Value are updated in real time. -; Example output: -; pool: www -; process manager: static -; start time: 01/Jul/2011:17:53:49 +0200 -; start since: 62636 -; accepted conn: 190460 -; listen queue: 0 -; max listen queue: 1 -; listen queue len: 42 -; idle processes: 4 -; active processes: 11 -; total processes: 15 -; max active processes: 12 -; max children reached: 0 -; -; By default the status page output is formatted as text/plain. Passing either -; 'html', 'xml' or 'json' in the query string will return the corresponding -; output syntax. Example: -; http://www.foo.bar/status -; http://www.foo.bar/status?json -; http://www.foo.bar/status?html -; http://www.foo.bar/status?xml -; -; By default the status page only outputs short status. Passing 'full' in the -; query string will also return status for each pool process. -; Example: -; http://www.foo.bar/status?full -; http://www.foo.bar/status?json&full -; http://www.foo.bar/status?html&full -; http://www.foo.bar/status?xml&full -; The Full status returns for each process: -; pid - the PID of the process; -; state - the state of the process (Idle, Running, ...); -; start time - the date and time the process has started; -; start since - the number of seconds since the process has started; -; requests - the number of requests the process has served; -; request duration - the duration in µs of the requests; -; request method - the request method (GET, POST, ...); -; request URI - the request URI with the query string; -; content length - the content length of the request (only with POST); -; user - the user (PHP_AUTH_USER) (or '-' if not set); -; script - the main script called (or '-' if not set); -; last request cpu - the %cpu the last request consumed -; it's always 0 if the process is not in Idle state -; because CPU calculation is done when the request -; processing has terminated; -; last request memory - the max amount of memory the last request consumed -; it's always 0 if the process is not in Idle state -; because memory calculation is done when the request -; processing has terminated; -; If the process is in Idle state, then informations are related to the -; last request the process has served. Otherwise informations are related to -; the current request being served. -; Example output: -; ************************ -; pid: 31330 -; state: Running -; start time: 01/Jul/2011:17:53:49 +0200 -; start since: 63087 -; requests: 12808 -; request duration: 1250261 -; request method: GET -; request URI: /test_mem.php?N=10000 -; content length: 0 -; user: - -; script: /home/fat/web/docs/php/test_mem.php -; last request cpu: 0.00 -; last request memory: 0 -; -; Note: There is a real-time FPM status monitoring sample web page available -; It's available in: ${prefix}/share/fpm/status.html -; -; Note: The value must start with a leading slash (/). The value can be -; anything, but it may not be a good idea to use the .php extension or it -; may conflict with a real PHP file. -; Default Value: not set -;pm.status_path = /status - -; The ping URI to call the monitoring page of FPM. If this value is not set, no -; URI will be recognized as a ping page. This could be used to test from outside -; that FPM is alive and responding, or to -; - create a graph of FPM availability (rrd or such); -; - remove a server from a group if it is not responding (load balancing); -; - trigger alerts for the operating team (24/7). -; Note: The value must start with a leading slash (/). The value can be -; anything, but it may not be a good idea to use the .php extension or it -; may conflict with a real PHP file. -; Default Value: not set -;ping.path = /ping - -; This directive may be used to customize the response of a ping request. The -; response is formatted as text/plain with a 200 response code. -; Default Value: pong -;ping.response = pong - -; The access log file -; Default: not set -;access.log = log/$pool.access.log - -; The access log format. -; The following syntax is allowed -; %%: the '%' character -; %C: %CPU used by the request -; it can accept the following format: -; - %{user}C for user CPU only -; - %{system}C for system CPU only -; - %{total}C for user + system CPU (default) -; %d: time taken to serve the request -; it can accept the following format: -; - %{seconds}d (default) -; - %{miliseconds}d -; - %{mili}d -; - %{microseconds}d -; - %{micro}d -; %e: an environment variable (same as $_ENV or $_SERVER) -; it must be associated with embraces to specify the name of the env -; variable. Some exemples: -; - server specifics like: %{REQUEST_METHOD}e or %{SERVER_PROTOCOL}e -; - HTTP headers like: %{HTTP_HOST}e or %{HTTP_USER_AGENT}e -; %f: script filename -; %l: content-length of the request (for POST request only) -; %m: request method -; %M: peak of memory allocated by PHP -; it can accept the following format: -; - %{bytes}M (default) -; - %{kilobytes}M -; - %{kilo}M -; - %{megabytes}M -; - %{mega}M -; %n: pool name -; %o: ouput header -; it must be associated with embraces to specify the name of the header: -; - %{Content-Type}o -; - %{X-Powered-By}o -; - %{Transfert-Encoding}o -; - .... -; %p: PID of the child that serviced the request -; %P: PID of the parent of the child that serviced the request -; %q: the query string -; %Q: the '?' character if query string exists -; %r: the request URI (without the query string, see %q and %Q) -; %R: remote IP address -; %s: status (response code) -; %t: server time the request was received -; it can accept a strftime(3) format: -; %d/%b/%Y:%H:%M:%S %z (default) -; %T: time the log has been written (the request has finished) -; it can accept a strftime(3) format: -; %d/%b/%Y:%H:%M:%S %z (default) -; %u: remote user -; -; Default: "%R - %u %t \"%m %r\" %s" -;access.format = %R - %u %t "%m %r%Q%q" %s %f %{mili}d %{kilo}M %C%% - -; The log file for slow requests -; Default Value: not set -; Note: slowlog is mandatory if request_slowlog_timeout is set -;slowlog = log/$pool.log.slow - -; The timeout for serving a single request after which a PHP backtrace will be -; dumped to the 'slowlog' file. A value of '0s' means 'off'. -; Available units: s(econds)(default), m(inutes), h(ours), or d(ays) -; Default Value: 0 -;request_slowlog_timeout = 0 - -; The timeout for serving a single request after which the worker process will -; be killed. This option should be used when the 'max_execution_time' ini option -; does not stop script execution for some reason. A value of '0' means 'off'. -; Available units: s(econds)(default), m(inutes), h(ours), or d(ays) -; Default Value: 0 -;request_terminate_timeout = 0 - -; Set open file descriptor rlimit. -; Default Value: system defined value -;rlimit_files = 1024 - -; Set max core size rlimit. -; Possible Values: 'unlimited' or an integer greater or equal to 0 -; Default Value: system defined value -;rlimit_core = 0 - -; Chroot to this directory at the start. This value must be defined as an -; absolute path. When this value is not set, chroot is not used. -; Note: you can prefix with '$prefix' to chroot to the pool prefix or one -; of its subdirectories. If the pool prefix is not set, the global prefix -; will be used instead. -; Note: chrooting is a great security feature and should be used whenever -; possible. However, all PHP paths will be relative to the chroot -; (error_log, sessions.save_path, ...). -; Default Value: not set -;chroot = - -; Chdir to this directory at the start. -; Note: relative path can be used. -; Default Value: current directory or / when chroot -chdir = / - -; Redirect worker stdout and stderr into main error log. If not set, stdout and -; stderr will be redirected to /dev/null according to FastCGI specs. -; Note: on highloaded environement, this can cause some delay in the page -; process time (several ms). -; Default Value: no -;catch_workers_output = yes - -; Limits the extensions of the main script FPM will allow to parse. This can -; prevent configuration mistakes on the web server side. You should only limit -; FPM to .php extensions to prevent malicious users to use other extensions to -; exectute php code. -; Note: set an empty value to allow all extensions. -; Default Value: .php -;security.limit_extensions = .php .php3 .php4 .php5 - -; Pass environment variables like LD_LIBRARY_PATH. All $VARIABLEs are taken from -; the current environment. -; Default Value: clean env -;env[HOSTNAME] = $HOSTNAME -;env[PATH] = /usr/local/bin:/usr/bin:/bin -;env[TMP] = /tmp -;env[TMPDIR] = /tmp -;env[TEMP] = /tmp - -; Additional php.ini defines, specific to this pool of workers. These settings -; overwrite the values previously defined in the php.ini. The directives are the -; same as the PHP SAPI: -; php_value/php_flag - you can set classic ini defines which can -; be overwritten from PHP call 'ini_set'. -; php_admin_value/php_admin_flag - these directives won't be overwritten by -; PHP call 'ini_set' -; For php_*flag, valid values are on, off, 1, 0, true, false, yes or no. - -; Defining 'extension' will load the corresponding shared extension from -; extension_dir. Defining 'disable_functions' or 'disable_classes' will not -; overwrite previously defined php.ini values, but will append the new value -; instead. - -; Note: path INI options can be relative and will be expanded with the prefix -; (pool, global or /usr) - -; Default Value: nothing is defined by default except the values in php.ini and -; specified at startup with the -d argument -;php_admin_value[sendmail_path] = /usr/sbin/sendmail -t -i -f www@my.domain.com -;php_flag[display_errors] = off -;php_admin_value[error_log] = /var/log/fpm-php.www.log -;php_admin_flag[log_errors] = on -;php_admin_value[memory_limit] = 32M diff --git a/cookbooks/dev/templates/default/fpm.conf.erb b/cookbooks/dev/templates/default/fpm.conf.erb deleted file mode 100644 index 523d8ca85..000000000 --- a/cookbooks/dev/templates/default/fpm.conf.erb +++ /dev/null @@ -1,47 +0,0 @@ -; DO NOT EDIT - This file is being maintained by Chef - -[<%= @user %>] -listen = 127.0.0.1:<%= @port %> -listen.backlog = 256 - -user = <%= @user %> -group = <%= @user %> - -pm = dynamic -pm.max_children = 10 -pm.start_servers = 4 -pm.min_spare_servers = 2 -pm.max_spare_servers = 6 -pm.max_requests = 10000 - -request_terminate_timeout = 1800 - -;pm.status_path = /status -;ping.path = /ping -;ping.response = pong - -;request_terminate_timeout = 0 -;request_slowlog_timeout = 0 -;slowlog = /var/log/php5-fpm.log.slow -;rlimit_files = 1024 -;rlimit_core = 0 -;chroot = -;chdir = /var/www -;catch_workers_output = yes - -env[HOSTNAME] = $HOSTNAME -env[PATH] = /usr/local/bin:/usr/bin:/bin -env[TMP] = /tmp -env[TMPDIR] = /tmp -env[TEMP] = /tmp - -php_admin_value[sendmail_path] = /usr/sbin/sendmail -t -i -f <%= @user %>@errol.openstreetmap.org -php_admin_value[open_basedir] = /home/<%= @user %>/:/tmp/:/usr/share/php/ -php_flag[display_errors] = on -;php_admin_value[error_log] = /var/log/fpm-php.www.log -;php_admin_flag[log_errors] = on -php_value[max_execution_time] = 300 -php_value[memory_limit] = 128M -php_value[post_max_size] = 32M -php_value[upload_max_filesize] = 32M - diff --git a/cookbooks/nominatim/attributes/default.rb b/cookbooks/nominatim/attributes/default.rb index fe1118822..1c0978afe 100644 --- a/cookbooks/nominatim/attributes/default.rb +++ b/cookbooks/nominatim/attributes/default.rb @@ -12,9 +12,9 @@ default[:nominatim][:enable_git_updates] = true default[:nominatim][:fpm_pools] = { :www => { - :port => "8000", + :port => 8000, :pm => "dynamic", - :max_children => "60" + :max_children => 60 } } diff --git a/cookbooks/nominatim/recipes/default.rb b/cookbooks/nominatim/recipes/default.rb index f0a365c09..0a9172b4b 100644 --- a/cookbooks/nominatim/recipes/default.rb +++ b/cookbooks/nominatim/recipes/default.rb @@ -364,9 +364,14 @@ end end node[:nominatim][:fpm_pools].each do |name, data| - php_fpm name do - template "fpm.conf.erb" - variables data.merge(:name => name) + php_fpm name.to_s do + port data[:port] + pm data[:pm] + pm_max_children data[:max_children] + pm_start_servers 20 + pm_min_spare_servers 10 + pm_max_spare_servers 20 + pm_max_requests 10000 end end diff --git a/cookbooks/nominatim/templates/default/fpm.conf.erb b/cookbooks/nominatim/templates/default/fpm.conf.erb deleted file mode 100644 index 386769efc..000000000 --- a/cookbooks/nominatim/templates/default/fpm.conf.erb +++ /dev/null @@ -1,22 +0,0 @@ -; DO NOT EDIT - This file is being maintained by Chef - -[<%= @name %>] -<% if @port -%> -listen = 127.0.0.1:<%= @port %> -<% else -%> -listen = /var/run/php5-fpm-<%= @name %>.sock -listen.owner = www-data -listen.group = www-data -<% end -%> - -user = www-data -group = www-data - -pm = <%= @pm %> -pm.max_children = <%= @max_children %> -pm.start_servers = 20 -pm.min_spare_servers = 10 -pm.max_spare_servers = 20 -pm.max_requests = 10000 - -security.limit_extensions = .php .phpx .phpj diff --git a/cookbooks/php/resources/fpm.rb b/cookbooks/php/resources/fpm.rb index 870ca4ff5..d70efe53c 100644 --- a/cookbooks/php/resources/fpm.rb +++ b/cookbooks/php/resources/fpm.rb @@ -20,19 +20,31 @@ default_action :create property :pool, :kind_of => String, :name_property => true -property :cookbook, :kind_of => String -property :template, :kind_of => String, :required => true -property :variables, :kind_of => Hash, :default => {} +property :port, :kind_of => Integer +property :user, :kind_of => String, :default => "www-data" +property :group, :kind_of => String, :default => "www-data" +property :pm, :kind_of => String, :default => "dynamic" +property :pm_max_children, :kind_of => Integer, :default => 5 +property :pm_start_servers, :kind_of => Integer, :default => 2 +property :pm_min_spare_servers, :kind_of => Integer, :default => 1 +property :pm_max_spare_servers, :kind_of => Integer, :default => 3 +property :pm_max_requests, :kind_of => Integer, :default => 500 +property :request_terminate_timeout, :kind_of => Integer, :default => 0 +property :environment, :kind_of => Hash, :default => {} +property :php_values, :kind_of => Hash, :default => {} +property :php_admin_values, :kind_of => Hash, :default => {} +property :php_flags, :kind_of => Hash, :default => {} +property :php_admin_flags, :kind_of => Hash, :default => {} property :reload_fpm, :kind_of => [TrueClass, FalseClass], :default => true action :create do - declare_resource :template, conf_file do - cookbook new_resource.cookbook - source new_resource.template + template conf_file do + cookbook "php" + source "pool.conf.erb" owner "root" group "root" mode "644" - variables new_resource.variables + variables new_resource.to_hash end end diff --git a/cookbooks/php/templates/default/pool.conf.erb b/cookbooks/php/templates/default/pool.conf.erb new file mode 100644 index 000000000..ad6894c9a --- /dev/null +++ b/cookbooks/php/templates/default/pool.conf.erb @@ -0,0 +1,45 @@ +; DO NOT EDIT - This file is being maintained by Chef + +[<%= @pool %>] +<% if @port -%> +listen = 127.0.0.1:<%= @port %> +listen.backlog = 256 +<% else -%> +listen = /run/php/<%= @pool %>.sock +listen.owner = www-data +listen.group = www-data +<% end -%> + +user = <%= @user %> +group = <%= @group %> + +pm = <%= @pm %> +pm.max_children = <%= @pm_max_children %> +pm.start_servers = <%= @pm_start_servers %> +pm.min_spare_servers = <%= @pm_min_spare_servers %> +pm.max_spare_servers = <%= @pm_max_spare_servers %> +pm.max_requests = <%= @pm_max_requests %> + +request_terminate_timeout = <%= @request_terminate_timeout %> + +security.limit_extensions = .php .phpx .phpj + +<% @environment.each do |name, value| -%> +env[<%= name %>] = <%= value %> +<% end -%> + +<% @php_values.each do |name, value| -%> +php_value[<%= name %>] = <%= value %> +<% end -%> + +<% @php_admin_values.each do |name, value| -%> +php_admin_value[<%= name %>] = <%= value %> +<% end -%> + +<% @php_flags.each do |name, value| -%> +php_flag[<%= name %>] = <%= value %> +<% end -%> + +<% @php_admin_flags.each do |name, value| -%> +php_admin_flag[<%= name %>] = <%= value %> +<% end -%>