Upgrade Postgres 9.2 to 9.6

.travis.yml

@@ -1,3 +1,4 @@
+dist: trusty # precise doesn't have pg 9.6 yet
 sudo: false
 branches:
   only:
@@ -21,7 +22,7 @@ language: erlang
 otp_release:
   - 18.3
 addons:
-  postgresql: "9.3"
+  postgresql: "9.6"
   apt:
     sources:
       - chef-stable-precise

RELEASE_NOTES.md

@@ -14,6 +14,31 @@ For prior releases, see [PRIOR\_RELEASE\_NOTES.md](PRIOR_RELEASE_NOTES.md).
   the secret being changed. With the optional flag `--with-restart`,
   `chef-server-ctl set-secret` will attempt to automatically restart the
   dependent services.
+* [Upgrade to PostgreSQL 9.6](https://github.com/chef/chef-server/pull/1310),
+  Chef Server now uses the latest stable version of the 9.6 series (9.6.3). Upgrades of existing
+  installations are done automatically, but creating backups is advised.
+
+  The information below only applies if you have set a custom value for `checkpoint_segments`
+  in your `/etc/opscode/chef-server.rb`. If you have not set a custom value, there is nothing to
+  change:
+
+  The `checkpoint_segments` configuration setting is gone, so if you before have set
+
+      postgresql['checkpoint_segments'] = 10
+
+  you now have to specify (for example, please consult the documentation linked below)
+
+      postgresql['max_wal_size'] = '3G'
+
+  The default setting for `max_wal_size` is 1G. [The PostgreSQL release notes](https://www.postgresql.org/docs/9.6/static/release-9-5.html) mention a conversion
+  rule: `max_wal_size = (3 * checkpoint_segments) * 16MB`. They also say that the default value
+  for `max_wal_size` (1GB) should fit in most settings, so this conversion is not done automatically.
+
+  This update also adds two further new configurables in the ["Checkpoints" group](https://www.postgresql.org/docs/9.6/static/runtime-config-wal.html#RUNTIME-CONFIG-WAL-CHECKPOINTS), `min_wal_size` and
+  `checkpoint_flush_after`.
+
+  As part of the upgrade procdure, running `chef-server-ctl cleanup` will remove Postgres 9.2's data
+  and logs.
 
 ## 12.15.8 (2017-06-20)
 

omnibus/config/software/oc_id.rb

@@ -27,7 +27,7 @@
 # and makeds more sense than adding the thing that is stopping
 # it from building on s390 (libxml2).
 dependency "nokogiri"
-dependency "postgresql92" # for libpq
+dependency "postgresql96" # for libpq
 dependency "nodejs-binary"
 dependency "ruby"
 dependency "bundler"

omnibus/config/software/partybus.rb

@@ -21,7 +21,7 @@
 license :project_license
 
 dependency "bundler"
-dependency "postgresql92"
+dependency "postgresql96"
 
 build do
   env = with_standard_compiler_flags(with_embedded_path)

omnibus/config/software/perl_pg_driver.rb

@@ -22,7 +22,7 @@
 
 dependency "perl"
 dependency "cpanminus"
-dependency "postgresql92"
+dependency "postgresql96"
 
 source url: "http://search.cpan.org/CPAN/authors/id/T/TU/TURNSTEP/DBD-Pg-#{version}.tar.gz",
        md5: "547de1382a47d66872912fe64282ff55"

omnibus/config/software/postgresql92-bin.rb

@@ -0,0 +1,57 @@
+#
+# Copyright 2017 Chef Software, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name "postgresql92-bin"
+
+default_version "9.2.21"
+
+license "PostgreSQL"
+license_file "COPYRIGHT"
+skip_transitive_dependency_licensing true
+
+dependency "zlib"
+dependency "openssl"
+dependency "libedit"
+dependency "ncurses"
+dependency "libossp-uuid"
+dependency "config_guess"
+
+source url: "https://ftp.postgresql.org/pub/source/v#{version}/postgresql-#{version}.tar.bz2"
+
+version("9.2.21") { source sha256: "0697e843523ee60c563f987f9c65bc4201294b18525d6e5e4b2c50c6d4058ef9" }
+
+relative_path "postgresql-#{version}"
+
+build do
+  env = with_standard_compiler_flags(with_embedded_path)
+  short_version = version.gsub(/^([0-9]+).([0-9]+).[0-9]+$/, '\1.\2')
+
+  update_config_guess(target: "config")
+
+  command "./configure" \
+          " --prefix=#{install_dir}/embedded/postgresql/#{short_version}" \
+          " --with-libedit-preferred" \
+          " --with-openssl" \
+          " --with-ossp-uuid" \
+          " --with-includes=#{install_dir}/embedded/include" \
+          " --with-libraries=#{install_dir}/embedded/lib", env: env
+
+  make "world -j #{workers}", env: env
+  make "-C src/interfaces/libpq install", env: env  # libpq.so
+  make "-C src/backend install-bin", env: env       # postgres binary
+  make "-C src/timezone/tznames  install", env: env # share/timezonesets
+  make "-C src/bin install", env: env               # pg_*, psql binaries
+end

omnibus/config/software/postgresql92.rb → omnibus/config/software/postgresql96.rb

@@ -1,5 +1,5 @@
 #
-# Copyright 2012-2017 Chef Software, Inc.
+# Copyright 2017 Chef Software, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,32 +14,34 @@
 # limitations under the License.
 #
 
-name "postgresql92"
-default_version "9.2.21"
+name "postgresql96"
+
+default_version "9.6.3"
 
 license "PostgreSQL"
 license_file "COPYRIGHT"
 skip_transitive_dependency_licensing true
 
-source url: "https://ftp.postgresql.org/pub/source/v9.2.21/postgresql-9.2.21.tar.bz2",
-       sha256: "0697e843523ee60c563f987f9c65bc4201294b18525d6e5e4b2c50c6d4058ef9"
-
 dependency "zlib"
 dependency "openssl"
 dependency "libedit"
 dependency "ncurses"
 dependency "libossp-uuid"
 dependency "config_guess"
 
-relative_path "postgresql-9.2.21"
+source url: "https://ftp.postgresql.org/pub/source/v#{version}/postgresql-#{version}.tar.bz2"
+version("9.6.3") { source sha256: "1645b3736901f6d854e695a937389e68ff2066ce0cde9d73919d6ab7c995b9c6" }
+
+relative_path "postgresql-#{version}"
 
 build do
   env = with_standard_compiler_flags(with_embedded_path)
+  short_version = version.gsub(/^([0-9]+).([0-9]+).[0-9]+$/, '\1.\2')
 
   update_config_guess(target: "config")
 
   command "./configure" \
-          " --prefix=#{install_dir}/embedded/postgresql/9.2" \
+          " --prefix=#{install_dir}/embedded/postgresql/#{short_version}" \
           " --with-libedit-preferred" \
           " --with-openssl" \
           " --with-ossp-uuid" \
@@ -49,13 +51,8 @@
   make "world -j #{workers}", env: env
   make "install-world -j #{workers}", env: env
 
-
-  # Postgres 9.2 is our "real" Postgres installation (prior versions
-  # that are installed are solely to facilitate upgrades).  As a
-  # result, we need to have the binaries for this version available
-  # with the other binaries used by Private Chef.
   block do
-    Dir.glob("#{install_dir}/embedded/postgresql/9.2/bin/*").sort.each do |bin|
+    Dir.glob("#{install_dir}/embedded/postgresql/#{short_version}/bin/*").sort.each do |bin|
       link bin, "#{install_dir}/embedded/bin/#{File.basename(bin)}"
     end
   end

omnibus/config/software/private-chef-cookbooks.rb

@@ -53,7 +53,8 @@
         run_list: [
           'recipe[private-chef::post_11_upgrade_cleanup]',
           'recipe[private-chef::post_12_upgrade_cleanup]',
-          'recipe[private-chef::solr4_gclog_cleanup]'
+          'recipe[private-chef::solr4_gclog_cleanup]',
+          'recipe[private-chef::postgres_upgrade_cleanup]'
         ]
       )
     end

omnibus/config/software/server-complete.rb

@@ -33,7 +33,8 @@
 dependency "erlang-crypto2"
 
 # the backend
-dependency "postgresql92"
+dependency "postgresql92-bin" # for upgrading 9.2 -> 9.6
+dependency "postgresql96"
 dependency "rabbitmq"
 dependency "redis" # dynamic routing controls
 dependency "opscode-solr4"

omnibus/files/private-chef-cookbooks/private-chef/attributes/default.rb

@@ -562,7 +562,7 @@
 # PostgreSQL
 ###
 # For now, we're hardcoding the version directory suffix here:
-default['private_chef']['postgresql']['version'] = "9.2"
+default['private_chef']['postgresql']['version'] = "9.6"
 # In the future, we're probably going to want to do something more elegant so we
 # don't accidentally overwrite this directory if we upgrade PG to 9.3: keeping these
 # directories straight is important because in the distant future (the year 2000)
@@ -593,7 +593,7 @@
 default['private_chef']['postgresql']['shmmax'] = 17179869184
 default['private_chef']['postgresql']['shmall'] = 4194304
 default['private_chef']['postgresql']['wal_level'] = "minimal"
-default['private_chef']['postgresql']['archive_mode'] = "off"
+default['private_chef']['postgresql']['archive_mode'] = "off" # "cannot be enabled when wal_level is set to minimal"
 default['private_chef']['postgresql']['archive_command'] = ""
 default['private_chef']['postgresql']['archive_timeout'] = 0 # 0 is disabled.
 
@@ -617,10 +617,19 @@
 default['private_chef']['postgresql']['shared_buffers'] = "#{(shared_bytes/1024).to_i}MB"
 
 default['private_chef']['postgresql']['work_mem'] = "8MB"
-default['private_chef']['postgresql']['effective_cache_size'] = "#{(node['memory']['total'].to_i / 2) / (1024)}MB"
-default['private_chef']['postgresql']['checkpoint_segments'] = 3
+default['private_chef']['postgresql']['effective_cache_size'] = "#{(node['memory']['total'].to_i / 2)/1024}MB"
+# Note: the checkpoint_segments setting was removed.
+# https://www.postgresql.org/docs/9.6/static/release-9-5.html says
+#   max_wal_size = (3 * checkpoint_segments) * 16MB
+# would be a usable conversion rule, but it also says the new setting's default
+# should be OK for most people. Since the conversion rule yields a value that
+# is so much smaller than the default, we don't do the conversion, but merely
+# allow for overriding the default setting.
+default['private_chef']['postgresql']['max_wal_size'] = "1GB"
+default['private_chef']['postgresql']['min_wal_size'] = "80MB"
 default['private_chef']['postgresql']['checkpoint_timeout'] = "5min"
 default['private_chef']['postgresql']['checkpoint_completion_target'] = 0.5
+default['private_chef']['postgresql']['checkpoint_flush_after'] = "256kB"
 default['private_chef']['postgresql']['checkpoint_warning'] = "30s"
 
 ###

omnibus/files/private-chef-cookbooks/private-chef/libraries/preflight_postgres_validator.rb

@@ -13,14 +13,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-class PostgresqlPreflightValidator < PreflightValidator
-
-  def initialize(node)
-    super
+require_relative "./warnings.rb"
 
-  end
+class PostgresqlPreflightValidator < PreflightValidator
+  # This check used to verify that the external PG version matches the version
+  # we ship. When we bumped the version we ship to 9.6, we haven't yet
+  # introduced any changes that _require_ 9.6. So, these constants reflect the
+  # actually required PG version.
+  REQUIRED_MAJOR = 9
+  REQUIRED_MINOR = 2
 
   def run!
+    warn_about_removed_attribute('checkpoint_segments')
     verify_unchanged_external_flag
 
     # Our additional validations only apply when a database server exists,
@@ -88,6 +92,16 @@ def verify_unchanged_external_flag
     end
   end
 
+  # When upgrading to 9.6, checkpoint_segments config was removed. The new
+  # configurables' defaults are such that they should not need tuning. However,
+  # we warn users that have changed the old attribute that their changes don't
+  # have any effect anymore.
+  def warn_about_removed_attribute(attr)
+    return unless cs_pg_attr[attr]
+
+    ChefServer::Warnings.warn err_unused_postgres_configurable(attr)
+  end
+
   def connectivity_validation
     # all nodes are expected to be able to reach the database node
     # and connect to it - let's make a connection intended to fail
@@ -170,16 +184,12 @@ def backend_verify_postgres_version(connection)
     # Make sure the server is a supported version.
     r = connection.exec("SHOW server_version;")
     v = r[0]['server_version']
-    major, minor = v.split(".")
-    # Load up our required major/minor:
-    # NOTE: our current entry in version-manifest.json is 'postgres92',effectively hardcoding the version. Necessary
-    # change is captured here: https://github.com/chef/chef-server/issues/441
-    manifest = JSON.parse(File.read("/opt/opscode/version-manifest.json"))
-    required_major, required_minor = manifest['software']['postgresql92']['locked_version'].split(".")
+    major, minor = v.split(".").map(&:to_i)
 
     # Note that we're looking for the same major, and using our minor as the minimum version
-    # This provides compatibility with external databases that use 9.3+ before we officially upgrade to it.
-    unless major == required_major and minor >= required_minor
+    # This provides compatibility with external databases that use < 9.6 before we make use
+    # of any features available in > 9.2.
+    unless major == REQUIRED_MAJOR and minor >= REQUIRED_MINOR
       fail_with err_CSPG014_bad_postgres_version(v)
     end
   end
@@ -298,7 +308,7 @@ def err_CSPG013_not_superuser
 
   def err_CSPG014_bad_postgres_version(ver)
 <<EOM
-CSPG014: Chef Server currently requires PostgreSQL version 9.2 or greater.
+CSPG014: Chef Server currently requires PostgreSQL version #{REQUIRED_MAJOR}.#{REQUIRED_MINOR} or greater.
          The database you have provided is running version #{ver}.
 
          See https://docs.chef.io/error_messages.html#cspg014-incorrect-version
@@ -340,4 +350,15 @@ def err_CSPG017_role_exists(username)
 EOM
 
   end
+
+  def err_unused_postgres_configurable(setting)
+<<EOM
+The setting
+
+    postgresql['#{setting}']
+
+is no longer supported by the version of PostgreSQL included in Chef
+Server. Please check the release notes for details.
+EOM
+  end
 end