diff --git a/Rakefile b/Rakefile index 58df3ec3..14f1c246 100644 --- a/Rakefile +++ b/Rakefile @@ -1,8 +1,2 @@ -# Rakefile for puppet-lint (https://github.com/rodjek/puppet-lint) -# Run: rake lint - -require 'puppet-lint/tasks/puppet-lint' -PuppetLint.configuration.with_filename = true -PuppetLint.configuration.send('disable_documentation') -PuppetLint.configuration.send('disable_class_parameter_defaults') -PuppetLint.configuration.send('disable_80chars') +require 'rubygems' +require 'puppetlabs_spec_helper/rake_tasks' diff --git a/manifests/config.pp b/manifests/config.pp index 43da22e4..8e7cc3ef 100644 --- a/manifests/config.pp +++ b/manifests/config.pp @@ -7,18 +7,4 @@ Python::Virtualenv <| |> -> Python::Pip <| |> Python::Virtualenv <| |> -> Python::Requirements <| |> - if $python::gunicorn { - Class['python::install'] -> Python::Gunicorn <| |> - - Python::Gunicorn <| |> ~> Service['gunicorn'] - - service { 'gunicorn': - ensure => running, - enable => true, - hasrestart => true, - hasstatus => false, - pattern => '/usr/bin/gunicorn', - } - } - } diff --git a/manifests/gunicorn.pp b/manifests/gunicorn.pp index 2548e95d..82330914 100644 --- a/manifests/gunicorn.pp +++ b/manifests/gunicorn.pp @@ -14,7 +14,7 @@ # Gunicorn mode. # wsgi|django. Default: wsgi # -# [*dir*] +# [*package_root*] # Application directory. # # [*bind*] @@ -28,12 +28,12 @@ # === Examples # # python::gunicorn { 'vhost': -# ensure => present, -# virtualenv => '/var/www/project1', -# mode => 'wsgi', -# dir => '/var/www/project1/current', -# bind => 'unix:/tmp/gunicorn.socket', -# environment => 'prod', +# ensure => present, +# virtualenv => '/var/www/project1', +# mode => 'wsgi', +# package_root => '/var/www/project1/current', +# bind => 'unix:/tmp/gunicorn.socket', +# environment => 'prod', # } # # === Authors @@ -41,26 +41,50 @@ # Sergey Stankevich # define python::gunicorn ( - $ensure = present, - $virtualenv = false, - $mode = 'wsgi', - $dir = false, - $bind = false, - $app_interface = 'wsgi', - $environment = false, + $reporting = false, + $app_interface = 'django', + $base = '/opt/wwc', + $bind = false, + $ensure = present, + $environment = false, + $mode = 'wsgi', + $package_root = '/opt/wwc/mitx', + $port = '8000', + $pre_start_commands = [], + $respawn_limit = false, + $script_name = '', + $service_enabled = 'running', + $settings_module = undef, + $timeout = '30', + $upstart_template = template('python/gunicorn/gunicorn.conf.erb'), + $max_requests = 0, + $user = 'www-data', + $virtualenv = false, + $workers = undef, + $wsgi_app = undef, + $edxapp = false, + $stacked = false, ) { - # Parameter validation - if ! $dir { - fail('python::gunicorn: dir parameter must not be empty') - } + include 'edx::newrelic' + include '::python' - file { "/etc/gunicorn.d/${name}": - ensure => $ensure, - mode => '0644', + file { "/etc/init/${name}.conf": + ensure => file, owner => 'root', group => 'root', - content => template('python/gunicorn.erb'), + mode => '0644', + notify => Service[$name], + content => $upstart_template, + require => Python::Pip['gunicorn'], + } + + service { $name: + ensure => $service_enabled, + provider => 'upstart', + require => File["/etc/init/${name}.conf"], + tag => release, + subscribe => Class['edx::newrelic'], } } diff --git a/manifests/init.pp b/manifests/init.pp index c7ab10f2..8d0768ae 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -33,11 +33,12 @@ $version = 'system', $dev = false, $virtualenv = false, + $virtualenv_location = '/opt/edx', $gunicorn = false ) { # Module compatibility check - $compatible = [ 'Debian', 'Ubuntu' ] + $compatible = [ 'Debian', 'Ubuntu', 'Amazon' ] if ! ($::operatingsystem in $compatible) { fail("Module is not compatible with ${::operatingsystem}") } @@ -47,4 +48,15 @@ include python::install include python::config + # Move this to here so it gets installed a single time in the virtualenv. If + # we ever want to split into multiple venvs then this will move back to + # gunicorn.pp and we'll enhance python::pip to take arguments so the name + # can be unique and not cause clashes with multiple instances of the lms + # define. + python::pip { 'gunicorn': + ensure => present, + virtualenv => $virtualenv_location, + require => Class['python::config'], + } + } diff --git a/manifests/install.pp b/manifests/install.pp index dec99b8a..7092dac8 100644 --- a/manifests/install.pp +++ b/manifests/install.pp @@ -21,11 +21,4 @@ package { 'python-virtualenv': ensure => $venv_ensure } - $gunicorn_ensure = $python::gunicorn ? { - true => present, - default => absent, - } - - package { 'gunicorn': ensure => $gunicorn_ensure } - } diff --git a/manifests/requirements.pp b/manifests/requirements.pp index b6c9b346..ee84cf35 100644 --- a/manifests/requirements.pp +++ b/manifests/requirements.pp @@ -23,13 +23,15 @@ # define python::requirements ( $virtualenv = 'system', - $proxy = false + $proxy = false, + $owner = 'www-data', + $group = 'www-data', ) { $requirements = $name $pip_env = $virtualenv ? { - 'system' => '`which pip`', + 'system' => '`/bin/which pip`', default => "${virtualenv}/bin/pip", } @@ -44,8 +46,8 @@ file { $requirements: ensure => present, mode => '0644', - owner => 'root', - group => 'root', + owner => $owner, + group => $group, replace => false, content => '# Puppet will install and/or update pip packages listed here', } @@ -58,10 +60,10 @@ } exec { "python_requirements_update_${name}": - command => "${pip_env} install ${proxy_flag} -Ur ${requirements}", - cwd => $virtualenv, + command => "${pip_env} install ${proxy_flag} -r ${requirements}", + cwd => $req_dir, refreshonly => true, - timeout => 1800, + timeout => 3600, subscribe => Exec["python_requirements_check_${name}"], } diff --git a/manifests/virtualenv.pp b/manifests/virtualenv.pp index f22f4e14..b8811c1d 100644 --- a/manifests/virtualenv.pp +++ b/manifests/virtualenv.pp @@ -58,8 +58,8 @@ exec { "python_virtualenv_${venv_dir}": command => "mkdir -p ${venv_dir} \ ${proxy_command} \ - && virtualenv -p `which ${python}` ${venv_dir} \ - && ${venv_dir}/bin/pip install ${proxy_flag} --upgrade distribute pip", + && virtualenv -p `/bin/which ${python}` ${venv_dir} --distribute \ + && ${venv_dir}/bin/pip install ${proxy_flag} --upgrade pip distribute", creates => $venv_dir, } diff --git a/spec/classes/python_spec.rb b/spec/classes/python_spec.rb new file mode 100644 index 00000000..7fa1bb48 --- /dev/null +++ b/spec/classes/python_spec.rb @@ -0,0 +1,4 @@ +require 'spec_helper' + +describe "python" do +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 00000000..dc7e9f4a --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,2 @@ +require 'rubygems' +require 'puppetlabs_spec_helper/module_spec_helper' diff --git a/templates/gunicorn.erb b/templates/gunicorn.erb deleted file mode 100644 index 2cf3eedd..00000000 --- a/templates/gunicorn.erb +++ /dev/null @@ -1,35 +0,0 @@ -CONFIG = { -<% if mode == 'django' -%> - 'mode': 'django', -<% else -%> - 'mode': 'wsgi', -<% end -%> -<% if virtualenv -%> - 'environment': { -<% if environment -%> - 'ENVIRONMENT': '<%= environment %>', -<% end -%> - 'PYTHONPATH': '<%= virtualenv %>' - }, -<% end -%> - 'working_dir': '<%= dir %>', - 'user': 'www-data', - 'group': 'www-data', -<% if virtualenv -%> - 'python': '<%= virtualenv %>/bin/python', -<% else -%> - 'python': '/usr/bin/python', -<% end -%> - 'args': ( -<% if !virtualenv and !bind -%> - '--bind=unix:/tmp/gunicorn-<%= name %>.socket', -<% elsif virtualenv and !bind -%> - '--bind=unix:<%= virtualenv %>/<%= name %>.socket', -<% else -%> - '--bind=<%= bind %>', -<% end -%> - '--workers=<%= @processorcount.to_i*2 %>', - '--timeout=30', - 'app:app', - ), -} diff --git a/templates/gunicorn/gunicorn.conf.erb b/templates/gunicorn/gunicorn.conf.erb new file mode 100644 index 00000000..e7ecebd2 --- /dev/null +++ b/templates/gunicorn/gunicorn.conf.erb @@ -0,0 +1,65 @@ +# gunicorn + +description "gunicorn server" +author "Calen Pennington " + +<% if scope.lookupvar('stacked') == false -%> +start on runlevel [2345] +stop on runlevel [!2345] +<% else -%> +<% if scope.lookupvar('title') != 'lms-xml' -%> +start on started edxapp +<% end -%> +stop on stopped edxapp +<% end -%> + +respawn +<% if scope.lookupvar('respawn_limit') != false -%> +respawn limit 3 30 +<% end -%> + +env PID=/var/tmp/<%= name %>.pid +env NEW_RELIC_CONFIG_FILE=<%= scope.lookupvar('base') %>/newrelic.ini +env NEWRELIC=<%= scope.lookupvar('virtualenv') %>/bin/newrelic-admin +<% if scope.lookupvar('workers') -%> +env WORKERS=<%= scope.lookupvar('workers') %> +<% else -%> +env WORKERS=<%= 4 * scope.lookupvar('::processorcount').to_i %> +<% end -%> +env PORT=<%= scope.lookupvar('port') %> +env LANG=en_US.UTF-8 +env DJANGO_SETTINGS_MODULE=<%= scope.lookupvar('settings_module') %> +env SERVICE_VARIANT="<%= scope.lookupvar('title') %>" + +<% if scope.lookupvar('stacked') == false -%> +pre-start script + find ${package_root} -user ${user} -type f -name '*.pyc' -delete || true + find /tmp -user ${user} -type d -name tmp*mako -exec rm -rf {} \\; || true +end script +<% end -%> + +chdir <%= scope.lookupvar('package_root') %> +<% if scope.lookupvar('app_interface') != 'python' -%> +setuid <%= scope.lookupvar('user') %> +<% end -%> + +<% case scope.lookupvar('app_interface') when 'django' -%> +exec <% if scope.lookupvar('edx::newrelic::reporting') == true -%>$NEWRELIC run-program <% end -%><%= scope.lookupvar('virtualenv') %>/bin/gunicorn_django --max-requests <%= scope.lookupvar('max_requests') %> -b 127.0.0.1:$PORT -w $WORKERS --timeout=<%= scope.lookupvar('timeout') %> --pythonpath=<%= scope.lookupvar('package_root') %> --settings=<%= scope.lookupvar('settings_module') %> +<% when 'wsgi' -%> +exec <% if scope.lookupvar('edx::newrelic::reporting') == true -%>$NEWRELIC run-program <% end -%><%= scope.lookupvar('virtualenv') %>/bin/gunicorn --max-requests <%= scope.lookupvar('max_requests') %> --preload -b 127.0.0.1:$PORT -w $WORKERS --timeout=<%= scope.lookupvar('timeout') %> --pythonpath=<%= scope.lookupvar('package_root') %> <%= scope.lookupvar('wsgi_app') %> +<% when 'python' -%> +exec su - www-data -c '<%= scope.lookupvar('virtualenv') %>/bin/python <%= scope.lookupvar('package_root') %>/<%= scope.lookupvar('script_name') %> >/dev/null 1>&1' +<% end -%> + +<% if (scope.lookupvar('title') == 'lms-xml') || (scope.lookupvar('title') == 'lms') || (scope.lookupvar('title') == 'lms-preview') -%> +post-start script + while true + do + if $(curl -s -i localhost:$PORT/heartbeat | egrep -q '200 OK'); then + break; + else + sleep 1; + fi + done +end script +<% end -%>