Merge pull request #443 from qld-gov-au/develop

Develop to master - prepare for Amazon Linux 2023

attributes/ckan.rb

@@ -22,12 +22,12 @@
 #
 
 default['datashades']['ckan_web']['endpoint'] = '/'
-default['datashades']['ckan_web']['packages'] = ['xml-commons', 'git', 'libxslt', 'libxslt-devel', 'libxml2', 'libxml2-devel', 'libxslt', 'libxslt-devel', 'gcc', 'gcc-c++', 'make', 'xalan-j2', 'unzip', 'policycoreutils-python', 'squid']
+default['datashades']['ckan_web']['packages'] = ['xml-commons', 'git', 'libxslt', 'libxslt-devel', 'libxml2', 'libxml2-devel', 'libxslt', 'libxslt-devel', 'gcc', 'gcc-c++', 'make', 'xalan-j2', 'unzip', 'squid']
 default['datashades']['ckan_web']['alternative_packages'] = [
-    ['postgresql94', 'postgresql'],
-    ['postgresql94-devel', 'postgresql-devel'],
+    ['postgresql', 'postgresql15'],
+    ['postgresql-devel', 'postgresql15-server-devel'],
     ['python27-devel', 'python3-devel'],
-    ['python27-pip', 'python3-pip'],
+    ['policycoreutils-python', 'policycoreutils-python-utils'],
 ]
 
 default['datashades']['ckan_web']['dbuser'] = 'ckan_default'
@@ -62,13 +62,13 @@
 default['datashades']['ckan_web']['wsgi']['processes'] = '1'
 default['datashades']['ckan_web']['wsgi']['threads'] = '15'
 
-default['datashades']['ckan_ext']['packages'] = ['geos-devel.x86_64']
+default['datashades']['ckan_ext']['packages'] = []
 
 default['datashades']['redis']['hostname'] = 'localhost'
 default['datashades']['redis']['port'] = '6379'
 
 # Postgres settings
 default['datashades']['postgres']['password'] = ''
 default['datashades']['postgres']['rds'] = 'true'
-default['datashades']['postgres']['packages'] = ['postgresql95-server', 'postgresql95', 'postgresql95-devel', 'libtool', 'libxml2-devel', 'geos-devel', 'postgis']
+default['datashades']['postgres']['packages'] = ['postgresql95-server', 'postgresql95', 'postgresql95-devel', 'libtool', 'libxml2-devel', 'postgis']
 default['datashades']['postgres']['version'] = '95'

attributes/default.rb

@@ -31,7 +31,12 @@
 
 default['datashades']['backup']['retention'] = '30'
 
-default['datashades']['core']['packages'] = ['nfs-utils', 'yum-cron', 'clamav', 'gcc', 'jq', 'java-1.8.0-openjdk', 'perl-Switch', 'perl-DateTime', 'perl-Sys-Syslog', 'perl-LWP-Protocol-https', 'perl-Digest-SHA.x86_64', 'git', 'telnet', 'supervisor']
+default['datashades']['core']['packages'] = ['nfs-utils', 'clamav', 'gcc', 'jq', 'perl-Switch', 'perl-DateTime', 'perl-Sys-Syslog', 'perl-LWP-Protocol-https', 'perl-Digest-SHA.x86_64', 'git', 'telnet', 'cronie']
+default['datashades']['core']['alternative_packages'] = [
+    ['postfix', 'sendmail'],
+    ['yum-cron', 'dnf-automatic'],
+    ['python27-pip', 'python3-pip'],
+]
 default['datashades']['core']['unwanted-packages'] = ['java-1.7.0-openjdk']
 
 default['datashades']['log_bucket'] = 'osssio-ckan-web-logs'

attributes/solr.rb

@@ -21,4 +21,4 @@
 # limitations under the License.
 #
 
-default['datashades']['solr']['packages'] = ['java-1.8.0-openjdk']
+default['datashades']['solr']['packages'] = ['java-1.8.0']

recipes/ckan-configure.rb

@@ -28,15 +28,3 @@
     EOS
     not_if "grep '# export PYTHON_INSTALL_LAYOUT' /etc/profile.d/python-install-layout.sh"
 end
-
-cookbook_file "/etc/logrotate.d/ckan" do
-    source "ckan-logrotate"
-end
-
-if not system("grep 'alias git=' ~/.bash_profile")
-    execute "Add CKAN Git alias to Bash" do
-        command <<-EOS
-            echo "alias git='sudo -u ckan git'" >> ~/.bash_profile
-        EOS
-    end
-end

recipes/ckan-deploy.rb

@@ -77,6 +77,8 @@
 # #pyOpenSSL 22.0.0 (2022-01-29) - dropped py2 support but has issues on py3 which stops harvester working
 # #pyOpenSSL 23.0.0 (2023-01-01) - required due to harvest:  Error: HTTP general exception: module 'lib' has no attribute 'SSL_CTX_set_ecdh_auto'
 execute "Pin pip versions" do
+	user service_name
+	group service_name
 	command "#{virtualenv_dir}/bin/pip install 'setuptools>=44.1.0' 'pyOpenSSL>=23.0.0'"
 end
 
@@ -176,3 +178,8 @@
 	cwd install_dir
 	command "#{virtualenv_dir}/bin/python setup.py compile_catalog -f --locale en_AU"
 end
+
+# Configure CKAN log processing
+cookbook_file "/etc/logrotate.d/ckan" do
+    source "ckan-logrotate"
+end

recipes/ckan-setup.rb

@@ -155,3 +155,12 @@
 	to "#{virtualenv_dir}/etc"
 	link_type :symbolic
 end
+
+# Add Bash alias to automatically use 'ckan' account for Git commands
+if not system("grep 'alias git=' ~/.bash_profile")
+    execute "Add CKAN Git alias to Bash" do
+        command <<-EOS
+            echo "alias git='sudo -u ckan git'" >> ~/.bash_profile
+        EOS
+    end
+end

recipes/ckanbatch-configure.rb

@@ -18,35 +18,51 @@
 
 include_recipe "datashades::ckan-configure"
 
-extra_disk = "/mnt/local_data"
-if ::File.directory?(extra_disk) then
-    swap_file = "#{extra_disk}/swapfile_2g"
-    bash "Add swap disk" do
-        code <<-EOS
-            dd if=/dev/zero of=#{swap_file} bs=1024 count=2M
-            chmod 0600 #{swap_file}
-            mkswap #{swap_file}
-        EOS
-        not_if { ::File.exist?(swap_file) }
-    end
+virtualenv_dir = "/usr/lib/ckan/default"
+ckan_cli = "#{virtualenv_dir}/bin/ckan_cli"
 
-    execute "Enable swap disk" do
-        command "swapon -s | grep '^#{swap_file} ' || swapon #{swap_file}"
+if not system('yum info supervisor')
+    service "ckan-worker" do
+        action [:enable, :start]
     end
 end
 
-template "/usr/local/bin/ckan-monitor-job-queue.sh" do
-    source 'ckan-monitor-job-queue.sh.erb'
-    owner 'root'
-    group 'root'
+# Run tracking update at 8:30am everywhere
+file "/etc/cron.d/ckan-tracking-update" do
+    content "30 8 * * * root /usr/local/bin/pick-job-server.sh && #{ckan_cli} tracking update >/dev/null 2>&1\n"
+    mode '0644'
+    owner "root"
+    group "root"
+end
+
+file "/etc/cron.hourly/ckan-email-notifications" do
+    content "/usr/local/bin/pick-job-server.sh && /usr/local/bin/ckan-email-notifications.sh > /dev/null 2>&1\n"
+    mode '0755'
+    owner "root"
+    group "root"
+end
+
+file "/etc/cron.daily/ckan-revision-archival" do
+    content "/usr/local/bin/pick-job-server.sh && /usr/local/bin/archive-resource-revisions.sh >/dev/null 2>&1\n"
     mode '0755'
+    owner "root"
+    group "root"
 end
 
 file "/etc/cron.d/ckan-worker" do
     content "*/5 * * * * root /usr/local/bin/pick-job-server.sh && /usr/local/bin/ckan-monitor-job-queue.sh >/dev/null 2>&1\n"
     mode '0644'
 end
 
+# Only set cron job for lower environments
+file '/etc/cron.hourly/ckan-tracking-update' do
+    content "/usr/local/bin/pick-job-server.sh && #{ckan_cli} tracking update >/dev/null 2>&1\n"
+    mode '0755'
+    owner "root"
+    group "root"
+    only_if { node['datashades']['version'] == 'DEV' || node['datashades']['version'] == 'TEST' }
+end
+
 # Make any other instances aware of us
 #
 execute "/usr/local/bin/pick-job-server.sh || true"

recipes/ckanbatch-deploy.rb

@@ -50,14 +50,42 @@
 # Create job worker config files.
 #
 
-cookbook_file "/etc/supervisord.d/supervisor-ckan-worker.ini" do
-    source "supervisor-ckan-worker.conf"
-    owner "root"
-    group "root"
-    mode "0744"
+if system('yum info supervisor')
+    cookbook_file "/etc/supervisord.d/supervisor-ckan-worker.ini" do
+        source "supervisor-ckan-worker.conf"
+        owner "root"
+        group "root"
+        mode "0744"
+    end
+else
+    # Create files with our preferred ownership to work around https://github.com/systemd/systemd/issues/14385
+    execute "Start job worker log file" do
+        user service_name
+        group service_name
+        command "touch /var/log/ckan/ckan-worker.log"
+    end
+    systemd_unit "ckan-worker.service" do
+        content({
+            Unit: {
+                Description: 'CKAN default job worker',
+                After: 'network-online.target'
+            },
+            Service: {
+                User: service_name,
+                ExecStart: '/usr/lib/ckan/default/bin/ckan_cli jobs worker',
+                Restart: 'on-failure',
+                StandardOutput: 'append:/var/log/ckan/ckan-worker.log',
+                StandardError: 'append:/var/log/ckan/ckan-worker.log'
+            },
+            Install: {
+                WantedBy: 'multi-user.target'
+            }
+        })
+        action [:create]
+    end
 end
 
-# Set up maintenance cron jobs
+# Set up maintenance scripts needed for cron jobs
 
 cookbook_file "/usr/local/bin/archive-resource-revisions.sql" do
     source "archive-resource-revisions.sql"
@@ -105,49 +133,9 @@
     mode "0755"
 end
 
-# Remove unwanted cron job
-file '/etc/cron.daily/ckan-tracking-update' do
-    action :delete
-end
-#
-# # Remove unwanted cron job from higher environments
-# file '/etc/cron.hourly/ckan-tracking-update' do
-#     action :delete
-#     not_if { node['datashades']['version'] == 'DEV' || node['datashades']['version'] == 'TEST' }
-# end
-
-# Only set cron job for lower environments
-file '/etc/cron.hourly/ckan-tracking-update' do
-    content "/usr/local/bin/pick-job-server.sh && #{ckan_cli} tracking update >/dev/null 2>&1\n"
-    mode '0755'
-    owner "root"
-    group "root"
-    only_if { node['datashades']['version'] == 'DEV' || node['datashades']['version'] == 'TEST' }
-end
-
-# Run tracking update at 8:30am everywhere
-file "/etc/cron.d/ckan-tracking-update" do
-    content "30 8 * * * root /usr/local/bin/pick-job-server.sh && #{ckan_cli} tracking update >/dev/null 2>&1\n"
-    mode '0644'
-    owner "root"
-    group "root"
-end
-
-file "/etc/cron.hourly/ckan-email-notifications" do
-    content "/usr/local/bin/pick-job-server.sh && /usr/local/bin/ckan-email-notifications.sh > /dev/null 2>&1\n"
-    mode '0755'
-    owner "root"
-    group "root"
-end
-
-file "/etc/cron.daily/ckan-revision-archival" do
-    content "/usr/local/bin/pick-job-server.sh && /usr/local/bin/archive-resource-revisions.sh >/dev/null 2>&1\n"
+template "/usr/local/bin/ckan-monitor-job-queue.sh" do
+    source 'ckan-monitor-job-queue.sh.erb'
+    owner 'root'
+    group 'root'
     mode '0755'
-    owner "root"
-    group "root"
-end
-
-service "supervisord restart" do
-    service_name "supervisord"
-    action [:stop, :start]
 end

recipes/ckanbatch-setup.rb

@@ -21,3 +21,20 @@
 include_recipe "datashades::default"
 
 include_recipe "datashades::ckan-setup"
+
+extra_disk = "/mnt/local_data"
+if ::File.directory?(extra_disk) then
+    swap_file = "#{extra_disk}/swapfile_2g"
+    bash "Add swap disk" do
+        code <<-EOS
+            dd if=/dev/zero of=#{swap_file} bs=1024 count=2M
+            chmod 0600 #{swap_file}
+            mkswap #{swap_file}
+        EOS
+        not_if { ::File.exist?(swap_file) }
+    end
+
+    execute "Enable swap disk" do
+        command "swapon -s | grep '^#{swap_file} ' || swapon #{swap_file}"
+    end
+end

recipes/ckanweb-configure.rb

@@ -22,3 +22,9 @@
 
 include_recipe "datashades::ckan-configure"
 include_recipe "datashades::nginx-configure"
+
+if not system('yum info supervisor')
+    service "ckan-uwsgi" do
+        action [:enable, :start]
+    end
+end