geOrchestra VM - manifest/default.pp provisionner

default.pp

include ::apt
include ::sudo
include ::collectd

openldap::server::module {'back_monitor':
  ensure => present,
}
openldap::server::database { 'cn=Monitor':
  ensure => present,
  backend => 'monitor',
  require => Openldap::Server::Module['back_monitor'],
}

collectd::plugin { ['cpu', 'memory', 'disk']: }
collectd::config::plugin { 'configure network plugin sending to logstash':
  plugin => 'network',
  settings => 'Server "127.0.0.1"',
}

# Doesn't work in a puppet apply context
# It requires a complete puppet server architecture :(


::collectd::config::plugin { 'apache status':
  plugin   => 'apache',
  settings => '
<Instance "localhost">
URL "http://localhost/server-status?auto"
</Instance>
',
}

::collectd::config::plugin {'postgresql monitoring':
  plugin   => 'postgresql',
  settings => inline_template('
<Database georchestra>
  Host "localhost"
  Port "5432"
  User "www-data"
  Password "www-data"
</Database>
'),
}

collectd::config::plugin { 'openldap':
  plugin   => 'openldap',
  settings => '
<Instance localhost>
URL "ldap://localhost:389"
</Instance>
',
}

collectd::config::plugin { 'tcpconns custom config':
  plugin   => 'tcpconns',
  settings => '
# apache
LocalPort 80
LocalPort 443

# proxycas
LocalPort 8280
LocalPort 8443

# georchestra
LocalPort 8080

# geoserver
LocalPort 8180

# openldap
LocalPort 389
LocalPort 636

# postgresql
LocalPort 5432
',
}

collectd::config::plugin { 'processes custom config':
  plugin   => 'processes',
  settings => '
CollectContextSwitch true

Process      "slapd"
ProcessMatch "apache"       "/usr/sbin/apache2"
ProcessMatch "postgres"     "postgres"
ProcessMatch "georchestra"  "java.*catalina\.base=/srv/tomcat/georchestra"
ProcessMatch "geoserver"    "java.*catalina\.base=/srv/tomcat/geoserver"
ProcessMatch "proxycas"     "java.*catalina\.base=/srv/tomcat/proxycas"
',
}

collectd::config::plugin { 'configure java plugin':
plugin => 'java',
settings => '
JVMArg "-verbose:jni"
JVMArg "-Djava.class.path=/usr/share/collectd/java/collectd-api.jar:/usr/share/collectd/java/generic-jmx.jar"
LoadPlugin "org.collectd.java.GenericJMX"
<Plugin "GenericJMX">
################
# MBean blocks #
################
# Number of classes being loaded.
<MBean "classes">
  ObjectName "java.lang:type=ClassLoading"
  InstancePrefix "class_loading"
  #InstanceFrom ""
  <Value>
    Type "gauge"
    InstancePrefix "loaded_classes"
    #InstanceFrom ""
    Table false
    Attribute "LoadedClassCount"
  </Value>
</MBean>
# Time spent by the JVM compiling or optimizing.
<MBean "compilation">
  ObjectName "java.lang:type=Compilation"
  InstancePrefix "compilation"
  #InstanceFrom ""
  <Value>
    Type "total_time_in_ms"
    InstancePrefix "compilation_time"
    #InstanceFrom ""
    Table false
    Attribute "TotalCompilationTime"
  </Value>
</MBean>
# Garbage collector information
<MBean "garbage_collector">
  ObjectName "java.lang:type=GarbageCollector,*"
  InstancePrefix "gc-"
  InstanceFrom "name"
  <Value>
    Type "invocations"
    #InstancePrefix ""
    #InstanceFrom ""
    Table false
    Attribute "CollectionCount"
  </Value>
  <Value>
    Type "total_time_in_ms"
    InstancePrefix "collection_time"
    #InstanceFrom ""
    Table false
    Attribute "CollectionTime"
  </Value>
</MBean>
# Generic heap/nonheap memory usage.
<MBean "memory">
  ObjectName "java.lang:type=Memory"
  #InstanceFrom ""
  InstancePrefix "memory"
  # Creates four values: committed, init, max, used
  <Value>
    Type "memory"
    #InstancePrefix ""
    #InstanceFrom ""
    Table true
    Attribute "HeapMemoryUsage"
    InstancePrefix "heap-"
  </Value>
  # Creates four values: committed, init, max, used
  <Value>
    Type "memory"
    #InstancePrefix ""
    #InstanceFrom ""
    Table true
    Attribute "NonHeapMemoryUsage"
    InstancePrefix "nonheap-"
  </Value>
</MBean>
# Memory usage by memory pool.
<MBean "memory_pool">
  ObjectName "java.lang:type=MemoryPool,*"
  InstancePrefix "memory_pool-"
  InstanceFrom "name"
  <Value>
    Type "memory"
    #InstancePrefix ""
    #InstanceFrom ""
    Table true
    Attribute "Usage"
  </Value>
</MBean>

### MBeans by Catalina / Tomcat ###
# The global request processor (summary for each request processor)
<MBean "catalina/global_request_processor">
  ObjectName "Catalina:type=GlobalRequestProcessor,*"
  InstancePrefix "global_request_processor-"
  InstanceFrom "name"
  <Value>
    Type "io_octets"
    InstancePrefix "global"
    #InstanceFrom ""
    Table false
    Attribute "bytesReceived"
    Attribute "bytesSent"
  </Value>
  <Value>
    Type "total_requests"
    InstancePrefix "global"
    #InstanceFrom ""
    Table false
    Attribute "requestCount"
  </Value>
  <Value>
    Type "total_time_in_ms"
    InstancePrefix "global-processing"
    #InstanceFrom ""
    Table false
    Attribute "processingTime"
  </Value>
</MBean>
# Details for each  request processor
<MBean "catalina/detailed_request_processor">
  ObjectName "Catalina:type=RequestProcessor,*"
  InstancePrefix "detailed_request_processor-"
  InstanceFrom "worker"
  <Value>
    Type "io_octets"
    #InstancePrefix ""
    InstanceFrom "name"
    Table false
    Attribute "bytesReceived"
    Attribute "bytesSent"
  </Value>
  <Value>
    Type "total_requests"
    #InstancePrefix ""
    InstanceFrom "name"
    Table false
    Attribute "requestCount"
  </Value>
  <Value>
    Type "total_time_in_ms"
    InstancePrefix "processing-"
    InstanceFrom "name"
    Table false
    Attribute "processingTime"
  </Value>
</MBean>
# Thread pool
<MBean "catalina/thread_pool">
  ObjectName "Catalina:type=ThreadPool,*"
  InstancePrefix "thread_pool-"
  InstanceFrom "name"
  <Value>
    Type "threads"
    InstancePrefix "total"
    #InstanceFrom ""
    Table false
    Attribute "currentThreadCount"
  </Value>
  <Value>
    Type "threads"
    InstancePrefix "running"
    #InstanceFrom ""
    Table false
    Attribute "currentThreadsBusy"
  </Value>
</MBean>

## Geonetwork mbeans ##
<MBean "geonetwork/webapp_metrics">
  ObjectName "geonetwork.org.fao.geonet.monitor.webapp:type=WebappMetricsFilter,*"
  InstancePrefix "webapp_metrics-"
  InstanceFrom "name"
  <Value>
    Type "total_requests"
    #InstancePrefix ""
    #InstanceFrom ""
    Table false
    Attribute "Count"
  </Value>
</MBean>
<MBean "geonetwork/log4j">
  ObjectName "geonetwork.org.apache.log4j:type=Appender,*"
  InstancePrefix "log4j-"
  InstanceFrom "name"
  <Value>
    Type "invocations"
    #InstancePrefix ""
    #InstanceFrom ""
    Table false
    Attribute "Count"
  </Value>
</MBean>

#####################
# Connection blocks #
#####################
<Connection>
  ServiceURL "service:jmx:rmi:///jndi/rmi://localhost:9010/jmxrmi"
  InstancePrefix "geoserver_"
  Collect "classes"
  Collect "compilation"
  Collect "garbage_collector"
  Collect "memory"
  Collect "memory_pool"
  Collect "catalina/global_request_processor"
  Collect "catalina/detailed_request_processor"
  Collect "catalina/thread_pool"
</Connection>
<Connection>
  ServiceURL "service:jmx:rmi:///jndi/rmi://localhost:9011/jmxrmi"
  InstancePrefix "proxycas_"
  Collect "classes"
  Collect "compilation"
  Collect "garbage_collector"
  Collect "memory"
  Collect "memory_pool"
  Collect "catalina/global_request_processor"
  Collect "catalina/detailed_request_processor"
  Collect "catalina/thread_pool"
</Connection>
<Connection>
  ServiceURL "service:jmx:rmi:///jndi/rmi://localhost:9012/jmxrmi"
  InstancePrefix "georchestra_"
  Collect "classes"
  Collect "compilation"
  Collect "garbage_collector"
  Collect "memory"
  Collect "memory_pool"
  Collect "catalina/global_request_processor"
  Collect "catalina/detailed_request_processor"
  Collect "catalina/thread_pool"
  Collect "geonetwork/webapp_metrics"
  Collect "geonetwork/log4j"
</Connection>
</Plugin>
'
}


# augeas modifications of *.properties files in /etc/georchestra
# courtesy of @raphink

define java::property (
    $ensure = 'present',
    $value = undef,
    $target,
    ) {
  case $ensure {
    'present': {
      $changes = "set ${name} ${value}"
    }

    'absent': {
      $changes = "rm ${name}"
    }
    default: {
               fail "Wrong value for \$ensure: ${ensure}"
    }
  }
  augeas { "java property ${title} in ${target}":
    incl    => $target,
    lens    => 'Properties.lns',
    changes => $changes,
  }
}

define georchestra::properties (
  $values,
  $target = $name,
) {
  create_resources('java::property', $values, { target => $target })
}

node default {
  # vagrant as sudoers with no password
  sudo::conf {"vagrant_can_do_whatever_it_wants":
    ensure  => present,
    content => "vagrant ALL=(root) NOPASSWD: ALL\n",
  }

  # geOrchestra debian repository
  apt::source { 'jessie-main-contrib-non-free':
             location => 'http://ftp.fr.debian.org/debian',
             repos    => 'main contrib non-free',
             ;
                'build-georchestra-org':
             location    => 'http://build.georchestra.org/debian',
             release     => 'jessie',
             repos       => 'main',
             include_src => false,
             key         => '70466AE690BFC64BCAD3294684D9C3019D3C11A3',
             key_source  => 'http://build.georchestra.org/debian/build.georchestra.org.asc',
  } ->
  exec { 'ensures-apt-get-update-after-having-reconfigured-apt':
    command => '/usr/bin/apt-get update',
  } ->
  # s/exim/postfix/ and some other useful packages
  package { 'exim4':
      ensure => absent
      ;
            [ 'git', 'postfix', 'vim', 'htop', 'lightdm', 'virtualbox-guest-x11',
              'xfce4', 'gnome-terminal', 'chromium', 'bundler', 'chromedriver',
              'dovecot-imapd', 'icedove', 'libnss3-tools',
              'georchestra-analytics',
              'georchestra-cas',
              'georchestra-catalogapp',
              'georchestra-downloadform',
              'georchestra-extractorapp',
              'georchestra-geonetwork3',
              'georchestra-geoserver',
              'georchestra-header',
              'georchestra-ldapadmin',
              'georchestra-mapfishapp',
              'georchestra-security-proxy',
            ]:
      ensure => present,
  } ->
  # Creates a postgres/postgis db
  ::postgresql::server::db { 'georchestra postgresql database':
    dbname   => 'georchestra',
    user     => 'www-data',
    password => postgresql_password('www-data', 'www-data'),
  } ->
  ::postgresql::server::role { 'geonetwork postgresql role':
    username  => 'geonetwork',
    password_hash => postgresql_password('geonetwork', 'www-data'),
  } ->
  ::postgresql::server::database_grant { 'geonetwork role on georchestra db':
    privilege => 'ALL',
    db        => 'georchestra',
    role      => 'geonetwork',
  } ->
  # Classical geOrchestra recipe with no customization
  class { 'georchestra': } ->
  # installs postgis
  ::postgresql::server::extension { 'postgis':
      package_name => 'postgis',
      ensure       => 'present',
      database     => 'georchestra',
  } ->
  # adjusts security-proxy default mappings
  georchestra::properties { '/etc/georchestra/security-proxy/targets-mapping.properties':
       values => {
           'target-mappings.properties.analytics' => {
               name  => 'analytics',
               value => 'http://127.0.1.1:8080/analytics/',
             },
           'target-mappings.properties.catalogapp' => {
               name  => 'catalogapp',
               value => 'http://127.0.1.1:8080/catalogapp/',
             },
            'target-mappings.properties.downloadform' => {
               name  => 'downloadform',
               value => 'http://127.0.1.1:8080/downloadform/',
             },
            'target-mappings.properties.extractorapp' => {
               name  => 'extractorapp',
               value => 'http://127.0.1.1:8080/extractorapp/',
             },
            'target-mappings.properties.geonetwork' => {
               name  => 'geonetwork',
               value => 'http://127.0.1.1:8080/geonetwork/',
             },
            'target-mappings.properties.geoserver' => {
               name  => 'geoserver',
               value => 'http://127.0.1.1:8180/geoserver/',
             },
            'target-mappings.properties.geofence' => {
               name  => 'geofence',
               value => 'http://127.0.1.1:8180/geofence/',
             },
            'target-mappings.properties.header' => {
               name  => 'header',
               value => 'http://127.0.1.1:8080/header/',
             },
            'target-mappings.properties.ldapadmin' => {
               name  => 'ldapadmin',
               value => 'http://127.0.1.1:8080/ldapadmin/',
             },
            'target-mappings.properties.mapfishapp' => {
               name  => 'mapfishapp',
               value => 'http://127.0.1.1:8080/mapfishapp/',
             },
            'target-mappings.properties.static' => {
               name  => 'static',
               value => 'http://127.0.1.1:8080/static/',
             },
       }
  } ->
  # Configures log4j to socket append apache-common style logs from the SP
  georchestra::properties { '/etc/georchestra/security-proxy/log4j/log4j.properties':
    values => {
       'log4j.logger.org.georchestra.security.statistics-combined' => {
           name  => 'log4j.logger.org.georchestra.security.statistics-common',
           value => 'INFO,NETWORKSOCKET',
       },
       'log4j.appender.NETWORKSOCKET' => {
           name  => 'log4j.appender.NETWORKSOCKET',
           value => 'org.apache.log4j.net.SocketAppender',
       },
       'log4j.appender.NETWORKSOCKET.Port' => {
           name  => 'log4j.appender.NETWORKSOCKET.Port',
           value => '4712',
       },
       'log4j.appender.NETWORKSOCKET.RemoteHost' => {
           name  => 'log4j.appender.NETWORKSOCKET.RemoteHost',
           value => 'localhost',
       },
    }
  } ->
  # creates symlinks (i.e. deploys the webapps) into each tomcat instances
  file { '/srv/tomcat/proxycas/webapps/ROOT.war': ensure => link, target => '/usr/share/lib/georchestra-security-proxy/ROOT-generic.war' } ->
  file { '/srv/tomcat/proxycas/webapps/cas.war': ensure => link, target => '/usr/share/lib/georchestra-cas/cas-generic.war' } ->
  # GDAL in georchestra tomcat instance
  file { '/srv/tomcat/georchestra/lib/gdal.jar':
    ensure => link,
    target => '/usr/share/java/gdal.jar',
  } ->
  file { '/srv/tomcat/georchestra/webapps/analytics.war': ensure => link, target => '/usr/share/lib/georchestra-analytics/analytics-generic.war' } ->
  file { '/srv/tomcat/georchestra/webapps/catalogapp.war': ensure => link, target => '/usr/share/lib/georchestra-catalogapp/catalogapp-generic.war' } ->
  file { '/srv/tomcat/georchestra/webapps/downloadform.war': ensure => link, target => '/usr/share/lib/georchestra-downloadform/downloadform-generic.war' } ->
  file { '/srv/tomcat/georchestra/webapps/extractorapp.war': ensure => link, target => '/usr/share/lib/georchestra-extractorapp/extractorapp-generic.war' } ->
  # geonetwork minimal datadir
  vcsrepo { '/srv/tomcat/georchestra/work/gn_data':
      ensure    => present,
      provider  => git,
      owner     => 'tomcat',
      group     => 'tomcat',
      revision  => 'gn3.0.x',
      source    => 'https://github.com/georchestra/geonetwork_minimal_datadir.git',
  } ->
  file { '/srv/tomcat/georchestra/webapps/geonetwork.war': ensure => link, target => '/usr/share/lib/georchestra-geonetwork3/geonetwork-generic.war' } ->
  # geoserver minimal datadir
  vcsrepo { '/srv/tomcat/geoserver0/work/gs_data':
      ensure   => present,
      provider => git,
      owner    => 'tomcat',
      group    => 'tomcat',
      source   => 'https://github.com/georchestra/geoserver_minimal_datadir.git',
  } ->
  # setting up JAI / ImageIO
  file { '/srv/tomcat/geoserver0/lib/jai_codec.jar':
    ensure => link,
    target => '/usr/share/java/jai_codec.jar',
  } ->
  file { '/srv/tomcat/geoserver0/lib/jai_core.jar':
    ensure => link,
    target => '/usr/share/java/jai_core.jar',
  } ->
  file { '/srv/tomcat/geoserver0/lib/jai_imageio.jar':
    ensure => link,
    target => '/usr/share/java/jai_imageio.jar',
  } ->
  file { '/srv/tomcat/geoserver0/lib/mlibwrapper_jai.jar':
    ensure => link,
    target => '/usr/share/java/mlibwrapper_jai.jar',
  } ->
  file { '/srv/tomcat/geoserver0/lib/clibwrapper_jiio.jar':
    ensure => link,
    target => '/usr/share/java/clibwrapper_jiio.jar',
  } ->
  file { '/srv/tomcat/geoserver0/webapps/geoserver.war': ensure => link, target => '/usr/share/lib/georchestra-geoserver/geoserver-generic.war' } ->
  file { '/srv/tomcat/georchestra/webapps/header.war': ensure => link, target => '/usr/share/lib/georchestra-header/header-generic.war' } ->
  file { '/srv/tomcat/georchestra/webapps/ldapadmin.war': ensure => link, target => '/usr/share/lib/georchestra-ldapadmin/ldapadmin-generic.war' } ->
  file { '/srv/tomcat/georchestra/webapps/mapfishapp.war': ensure => link, target => '/usr/share/lib/georchestra-mapfishapp/mapfishapp-generic.war' } ->
  # Authorizes georchestra to sudo su without password
  sudo::conf {"georchestra_can_do_whatever_it_wants":
    ensure  => present,
    content => "georchestra ALL=(root) NOPASSWD: ALL\n",
  } ->
  # Auto login as georchestra user on lightdm
  augeas { "property autologin-user in /etc/lightdm/lightdm.conf":
    context => '/files/etc/lightdm/lightdm.conf',
    changes => ['set SeatDefaults/autologin-user georchestra'],
  } ->
  # reconfigure dovecot-imapd to allow plaintext passwords
  augeas { "enable plaintext passwords on dovecot-imapd":
    context => '/files/etc/dovecot/conf.d/10-auth.conf',
    changes => ['set disable_plaintext_auth no'],
  } ->
  # reconfigure dovecot to set privileged group to mail
  augeas { "set dovecot privileged_group to mail":
    context => '/files/etc/dovecot/conf.d/10-mail.conf',
    changes => ['set mail_privileged_group mail'],
  } ->
  file { 'we_we_we_georchestra_private_shall_exist':
    path => '/var/www/georchestra/private/',
    ensure => 'directory',
  } ->
  file { 'we_we_we_georchestra_conf_shall_exist':
    path => '/var/www/georchestra/conf/',
    ensure => 'directory',
  } ->

  # Setting up logstash
  class { 'logstash':
    package_url => 'https://download.elastic.co/logstash/logstash/packages/debian/logstash_1.5.4-1_all.deb',
  }
  logstash::configfile { 'logstash-config-for-geor-logging':
    content => "
        input {
                log4j {
                        mode => server
                        port => 4712
                        type => \"sp-common-logs\"
                }
                udp {
                        port => 25826
                        buffer_size => 1452
                        type => \"collectd\"
                        codec => collectd { }
                }
        }
        filter {
                if [type] == \"sp-common-logs\" {
                        grok { match => [ \"message\", \"%{COMMONAPACHELOG} (?<time.needed>(%{BASE10NUM}))\" ] }
                        mutate {
                                convert => {
                                        \"time.needed\" => \"float\"
                                        \"response\"    => \"integer\"
                                        \"bytes\"       => \"integer\"
                                }
                        }
                }
        }
        output {
                elasticsearch {}
        }
    ",
  }
  # setting up Elasticsearch
  apt::key { 'key-repo-el-1.7':
    key        => '46095ACC8548582C1A2699A9D27D666CD88E42B4',
    key_source => 'https://packages.elastic.co/GPG-KEY-elasticsearch',
  } ->
  apt::source { 'elasticsearch-1.7':
    location          => 'http://packages.elastic.co/elasticsearch/1.7/debian',
    release           => 'stable',
    repos             => 'main',
    include_src       => false,
    required_packages => 'debian-keyring debian-archive-keyring',
  } ->
  class { 'elasticsearch': }
  elasticsearch::instance { 'es-01': }

  # kibana4
  class { '::kibana4':
    package_ensure    => '4.1.1-linux-x64',
    package_provider  => 'archive',
    symlink           => false,
    manage_user       => true,
    kibana4_user      => kibana4,
    kibana4_group     => kibana4,
    kibana4_gid       => 200,
    kibana4_uid       => 200,
    elasticsearch_url => 'http://localhost:9200',
  } ->
  file { 'kibana_configuration_for_nginx':
    path    => '/var/www/georchestra/conf/kibana.conf',
    content => "location /kibana/ {
  proxy_pass http://127.0.0.1:5601/;
}",
    notify  => Class['::nginx::service'],
  }

  # Checks out the capybara-based geOrchestra test suite
  vcsrepo { '/home/georchestra/runtime-tests':
      ensure   => present,
      provider => git,
      owner    => 'georchestra',
      group    => 'georchestra',
      source   => 'https://github.com/pmauduit/georchestra_runtime_tests.git',
  }

}

hiera-common.yaml

---
georchestra::directory::suffix: dc=georchestra,dc=org
georchestra::directory::rootdn: cn=admin,dc=georchestra,dc=org
georchestra::directory::rootpw: 'secret'
georchestra::webserver::type: 'nginx'
georchestra::globals::extra_tomcat_setenv:
  - 'ADD_JAVA_OPTS="${ADD_JAVA_OPTS} -Dgeorchestra.datadir=/etc/georchestra" '
  - 'ADD_JAVA_OPTS="${ADD_JAVA_OPTS} -Dgeonetwork.dir=/srv/tomcat/georchestra/work/gn_data" '
  - 'ADD_JAVA_OPTS="${ADD_JAVA_OPTS} -Dgeonetwork.schema.dir=/srv/tomcat/georchestra/work/gn_data/config/schema_plugins" '
  - 'ADD_JAVA_OPTS="${ADD_JAVA_OPTS} -Dgeonetwork.jeeves.configuration.overrides.file=/etc/georchestra/geonetwork/config/config-overrides-georchestra.xml" '
  - 'JAVA_XMX=1024m '
georchestra::globals::extra_tomcat_proxycas_setenv:
  - 'ADD_JAVA_OPTS="${ADD_JAVA_OPTS} -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9011 -Dcom.sun.management.jmxremote.local.only=true"'
  - 'ADD_JAVA_OPTS="${ADD_JAVA_OPTS} -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"'
georchestra::globals::extra_tomcat_georchestra_setenv:
  - 'ADD_JAVA_OPTS="${ADD_JAVA_OPTS} -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9012 -Dcom.sun.management.jmxremote.local.only=true"'
  - 'ADD_JAVA_OPTS="${ADD_JAVA_OPTS} -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"'