autobuild-standalone

#!/usr/bin/env ruby
# encoding: utf-8
abort "This tool requires Ruby 1.9" if RUBY_VERSION <= '1.9'

require 'optparse'
require 'logger'
require 'fileutils'
require 'date'
require 'shellwords'
require_relative './library'

ROOT = File.expand_path(File.dirname(__FILE__))

class Autobuild
  DEFAULT_RUBY_VERSION = '2.2.3'
  APACHE_VERSIONS = ['2.0', '2.2', '2.4']

  include FileUtils
  include RubyLibrary

  def initialize
    @output_files = []
  end

  def start
    parse_options
    set_environment
    initialize_logger
    fetch_latest_code
    inspect_codebase
    fetch_nginx
    if already_built?(output_commit_subdir)
      log "Already built commit #{@commit}."
    else
      recompile_passenger
      recompile_nginx
      begin
        create_build_output
        test_build_output
      rescue Exception
        remove_build_output
        raise
      end
      commit_build_output
    end
    setup_index_symlinks
    create_release_file
  end

private
  def parse_options
    @options = {
      :repo_dir => File.expand_path("repo"),
      :output_dir => File.expand_path("output"),
      :with_ngx_geoip_module => true,
      :with_ruby_extensions => true,
      :with_apache_modules => true,
      :static_libs_prefix => "/usr/local/override"
    }
    parser = OptionParser.new do |opts|
      nl = "\n#{' ' * 37}"
      opts.banner = "Usage: ./autobuild-standalone OPTIONS"
      opts.separator ""
      
      opts.separator "Options:"
      opts.on("--git-url URL", String, "The git URL") do |val|
        @options[:git_url] = val
      end
      opts.on("--repo-dir DIR", String, "The directory in which to store the repository") do |val|
        @options[:repo_dir] = File.expand_path(val)
      end
      opts.on("--tag NAME", String, "Checkout a specific git tag. Implies --create-release-symlink") do |val|
        @options[:commit] = val
        @options[:create_release_symlink] = true
      end
      opts.on("--branch NAME", String, "Checkout a specific branch") do |val|
        @options[:commit] = val
      end
      opts.on("--create-release-symlink", "Create a release symlink after building") do
        @options[:create_release_symlink] = true
      end
      opts.on("--output-dir DIR", String, "The directory in which to store binaries") do |val|
        @options[:output_dir] = File.expand_path(val)
      end
      opts.on("--work-tag NAME", String, "A tag to prevent duplicate builts. A file in " +
        "the output directory will be created using this tag name") do |val|
        @options[:work_tag] = val
      end
      opts.on("--result-file FILE", String, "Save output filenames to this file") do |val|
        @options[:result_file] = val
      end
      opts.on("--metadata-result-file FILE", String, "Save git commit and other info to this file") do |val|
        @options[:metadata_result_file] = val
      end
      opts.on("--without-ngx_geoip_module", "Do not build Nginx geoip_module") do
        @options[:with_ngx_geoip_module] = false
      end
      opts.on("--without-ruby-extensions", "Do not build Ruby extensions") do
        @options[:with_ruby_extensions] = false
      end
      opts.on("--without-apache-modules", "Do not build Apache modules") do
        @options[:with_apache_modules] = false
      end
      opts.on("--static-libs-prefix PREFIX", String, "Prefix in which static libraries like libssl.a are located") do |val|
        @options[:static_libs_prefix] = val
      end
      opts.on("--timeout TIMEOUT", Integer, "Timeout in seconds") do |val|
        @options[:timeout] = val
      end
      opts.on("--dry-run", "Do everything exception compiling") do
        @options[:dry_run] = true
      end
      opts.on("--help", "-h", "Show help message") do
        @options[:help] = true
      end
    end

    begin
      parser.parse!
    rescue OptionParser::ParseError => e
      STDERR.puts e
      STDERR.puts
      STDERR.puts "Please see './autobuild --help' for valid options."
      exit 1
    end

    if @options[:help]
      puts parser
      exit 0
    end
    if !@options[:git_url]
      abort "Please specify --git-url."
    end
  end

  def set_environment
    STDOUT.sync = STDERR.sync = true
    if !cc_uses_ccache?
      set_env('USE_CCACHE', 1)
    end
    set_env('GZIP', '--best')
    set_env('VERBOSE', 1)
    if @options[:timeout]
      Process.setpgrp
      Thread.new do
        Thread.current.abort_on_exception = true
        sleep(@options[:timeout])
        STDERR.puts "*** TIMEOUT #{@options[:timeout]} SECONDS ***"
        Process.kill('-TERM', Process.pid)
      end
    end
  end

  def cc_uses_ccache?
    (cc = ENV['CC']) && (cc =~ /^ccache /)
  end

  def set_env(key, value)
    log "export #{key}=#{value}"
    ENV[key] = value.to_s
  end

  def initialize_logger
    @logger = Logger.new(STDOUT)
  end

  def fetch_latest_code
    if File.exist?(passenger_code_dir)
      git_fetch_and_reset
    else
      git_clone
    end
    sh "git", "submodule", "update", "--init", "--recursive"
  end

  def git_fetch_and_reset
    cd_repo_dir
    sh "rm -rf *"
    sh "git fetch"
    reset_to_requested_commit
  end

  def git_clone
    mkdir_p @options[:repo_dir], :verbose => true
    sh "git", "clone", @options[:git_url], passenger_code_dir
    cd_repo_dir
    reset_to_requested_commit
  end

  def reset_to_requested_commit
    if @options[:commit]
      sh "git reset --hard '#{@options[:commit]}'"
    else
      sh "git reset --hard origin/master"
    end
    sh "git submodule update --init --recursive"
  end

  def cd_repo_dir
    log "cd #{passenger_code_dir}"
    Dir.chdir(passenger_code_dir)
  end

  def inspect_codebase
    lines = `git show --pretty='format:%h\n%H\n%s' -s`.split("\n")
    @commit, @sha, @subject = lines
    log "Commit #{@commit}: #{@subject}"
    if File.exist?("lib/phusion_passenger.rb")
      code = File.read("lib/phusion_passenger.rb")
    else
      code = File.read("src/ruby_supportlib/phusion_passenger.rb")
    end
    code =~ /[\t\s]VERSION_STRING *= *'(.+)'/
    @passenger_version = $1
    abort "Cannot infer Phusion Passenger version" if @passenger_version.to_s.empty?
    code =~ /PREFERRED_NGINX_VERSION *= *'(.+)'/
    @preferred_nginx_version = $1
    abort "Cannot infer preferred Nginx version" if @preferred_nginx_version.to_s.empty?

    @nginx_addon_dir = `bin/passenger-config about nginx-addon-dir`.strip
    abort "Cannot infer Nginx addon directory" if @nginx_addon_dir.empty?
  end

  def fetch_nginx
    log "Downloading Nginx #{@preferred_nginx_version}"
    if File.exist?(nginx_tarball)
      log "Tarball #{nginx_tarball} already exists. Skipping download."
    else
      sh "rm", "-f", nginx_tarball, nginx_tarball + ".asc"
      sh "wget", nginx_url, "-O", nginx_tarball
      sh "wget", nginx_url + ".asc", "-O", nginx_tarball + ".asc"
      begin
        sh "gpg", "--verify", nginx_tarball + ".asc"
      rescue SystemExit
        sh "rm", "-f", nginx_tarball, nginx_tarball + ".asc"
        raise "Cannot verify Nginx tarball signature"
      end
    end
  end

  def already_built?(dir)
    @options[:work_tag] && File.exist?(work_tag_filename(dir))
  end

  def work_tag_filename(dir)
    if @options[:work_tag]
      "#{dir}/built-#{@options[:work_tag]}.txt"
    else
      nil
    end
  end

  def recompile_passenger
    return if @options[:dry_run]

    if @options[:with_ruby_extensions]
      sh "~/.rvm/bin/rvm-exec ruby-1.8.7 env NOEXEC_DISABLE=1 rake native_support"
      sh "~/.rvm/bin/rvm-exec ruby-1.9.2 env NOEXEC_DISABLE=1 rake native_support"
      sh "~/.rvm/bin/rvm-exec ruby-1.9.3 env NOEXEC_DISABLE=1 rake native_support"
      sh "~/.rvm/bin/rvm-exec ruby-2.0.0 env NOEXEC_DISABLE=1 rake native_support"
      sh "~/.rvm/bin/rvm-exec ruby-2.1.7 env NOEXEC_DISABLE=1 rake native_support"
      sh "~/.rvm/bin/rvm-exec ruby-2.2.3 env NOEXEC_DISABLE=1 rake native_support"
    end

    sh "~/.rvm/bin/rvm-exec #{DEFAULT_RUBY_VERSION} #{drake} nginx OPTIMIZE=true"

    if @options[:with_apache_modules]
      APACHE_VERSIONS.each do |version|
        sh "rm -rf buildout/cache"
        sh "~/.rvm/bin/rvm-exec #{DEFAULT_RUBY_VERSION} #{drake} apache2 " +
          "APACHE2_OUTPUT_DIR=buildout/apache-#{version} " +
          "APXS2=/usr/local/apache-#{version}/bin/apxs " +
          "PATH=/usr/local/apache-#{version}/bin:#{ENV['PATH']}"
      end
    end

    # The .o files were only used for linking agent executables.
    sh "rm -f buildout/agents/*.o"
    sh "rm -rf buildout/agents/*.dSYM"
    sh "rm -f buildout/support-binaries/*.o"
    sh "rm -rf buildout/support-binaries/*.dSYM"
    # The .o and .a files were only used for linking the Apache module.
    sh "rm -rf buildout/apache*/*.o buildout/apache*/*.a buildout/apache*/module*"
    # The directory is no longer necessary, only libboost_oxt.a matters.
    sh "rm -rf buildout/common/libboost_oxt"
    # libeio and libev were only used for linking agent executables.
    sh "rm -rf buildout/libeio buildout/libev"
    sh "rm", "-f", *Dir["buildout/ruby/*/{Makefile,mkmf.log,*.o}"]
  end

  def recompile_nginx
    return if @options[:dry_run]

    # Nginx's OpenSSL detection is broken w.r.t. static OpenSSL libraries.
    if RUBY_PLATFORM =~ /darwin/
      env = {
        'CC' => ENV['CC'] + ' -lz'
      }
    else
      env = {
        'CC' => ENV['CC'] + ' -lz -ldl'
      }
    end

    Dir.chdir("buildout") do
      sh "tar", "xzvf", nginx_tarball
      Dir.chdir(nginx_basename) do
        command = "./configure " <<
          "--prefix=/tmp " <<
          "--with-cc-opt='-Wno-error' " <<
          "--with-http_ssl_module " <<
          "--with-http_v2_module " <<
          "--with-http_realip_module " <<
          "--with-http_gzip_static_module " <<
          "--with-http_stub_status_module " <<
          "--with-http_addition_module "
        if @options[:with_ngx_geoip_module]
          command << "--with-http_geoip_module "
        end
        command << "--add-module=#{@nginx_addon_dir}"
        sh env, command

        # We want to link statically to OpenSSL, PCRE and zlib.
        static_libs_prefix = @options[:static_libs_prefix]
        if RUBY_PLATFORM =~ /darwin/
          inline_sed "s|-lcrypto|#{static_libs_prefix}/lib/libcrypto/*.o|g", "objs/Makefile"
          inline_sed "s|-lssl|#{static_libs_prefix}/lib/libssl/*.o|g", "objs/Makefile"
          inline_sed "s|-lz$|#{static_libs_prefix}/lib/libz.a|g", "objs/Makefile"
          inline_sed "s|-lz |#{static_libs_prefix}/lib/libz.a |g", "objs/Makefile"
          inline_sed "s|-lpcre|#{static_libs_prefix}/lib/libpcre.a|g", "objs/Makefile"
        else
          inline_sed "s|-lcrypto|#{static_libs_prefix}/lib/libcrypto.a|g", "objs/Makefile"
          inline_sed "s|-lssl|#{static_libs_prefix}/lib/libssl.a|g", "objs/Makefile"
          inline_sed "s|-lz$|/usr/lib/libz.a|g", "objs/Makefile"
          inline_sed "s|-lz |/usr/lib/libz.a |g", "objs/Makefile"
          inline_sed "s|-lpcre|/usr/lib/libpcre.a|g", "objs/Makefile"
          inline_sed "s|-lGeoIP|/usr/lib/libGeoIP.a|g", "objs/Makefile"
        end
        
        sh "make"
        sh "#{strip_all} objs/nginx"
        sh "mv objs/nginx objs/nginx-#{@preferred_nginx_version}"
      end
    end
  end

  def create_build_output
    log "Creating build output directory #{output_save_subdir}"
    make_output_subdir(output_save_subdir)
    build_tarballs(output_save_subdir)
    create_binary_compatibility_symlinks(output_save_subdir)
    if @options[:work_tag]
      sh "touch", work_tag_filename(output_save_subdir)
    end
  end

  def build_tarballs(output_dir)
    log "Saving binaries to #{output_dir}"
    if @options[:dry_run]
      log "--dry-run given, not creating tarballs"
    else
      if @options[:with_ruby_extensions]
        Dir["buildout/ruby/*"].each do |subdir|
          name = File.basename(subdir)
          maketar "#{output_dir}/rubyext-#{name}.tar.gz", ["."], "buildout/ruby/#{name}"
        end
      end

      maketar "#{output_dir}/agent-#{binary_compat_id}.tar.gz",
        ["PassengerAgent"], "buildout/support-binaries"

      if @options[:with_apache_modules]
        APACHE_VERSIONS.each do |version|
          maketar "#{output_dir}/apache-#{version}-#{binary_compat_id}.tar.gz",
            ["mod_passenger.so"], "buildout/apache-#{version}"
        end
      end

      maketar "#{output_dir}/nginx-#{@preferred_nginx_version}-#{binary_compat_id}.tar.gz",
        ["nginx-#{@preferred_nginx_version}"], "buildout/#{nginx_basename}/objs"
    end
  end

  def create_binary_compatibility_symlinks(dir)
    if compatible_targets = ENV['MACOSX_COMPATIBLE_DEPLOYMENT_TARGETS']
      compatible_targets = compatible_targets.split
      results = []

      Dir.chdir(dir) do
        Dir["*.tar.gz"].each do |filename|
          compatible_targets.each do |target|
            new_filename = filename.sub(binary_compat_id, "x86_64-macosx-#{target}")
            if new_filename != filename
              log "Creating symlink #{new_filename}"
              if File.symlink?(new_filename)
                File.unlink(new_filename)
              end
              # Otherwise, if file already exists but is not a symlink,
              # let it raise an error.
              File.symlink(filename, new_filename)
            end
          end
        end
      end
    end
  end

  def test_build_output
    log "Testing Passenger"
    sh "mkdir", "-p", "download_cache"
    sh "cp -pR '#{output_save_subdir}'/*.tar.gz download_cache/"
    sh "ls -l download_cache/"
    sh_retry "gem install rack --no-rdoc --no-ri"
    if RUBY_PLATFORM =~ /linux/
      sh "rspec -f s -c --tty test/integration_tests/downloaded_binaries_tests.rb"
    else
      sh "bundle exec rspec -f s -c --tty test/integration_tests/downloaded_binaries_tests.rb"
    end
  end

  def commit_build_output
    log "Committing build output to #{output_commit_subdir}"
    make_output_subdir(output_commit_subdir)
    filenames = move_all_files(output_save_subdir, output_commit_subdir)
    sh "rmdir", output_save_subdir
    create_result_file(filenames.grep(/\.tar\.gz$/))
    create_metadata_result_file(output_commit_subdir)
  end

  def move_all_files(from, to)
    result = []
    Dir["#{from}/*"].each do |filename|
      basename = File.basename(filename)
      new_filename = "#{to}/#{basename}"
      sh "mv", filename, new_filename
      result << new_filename
    end
    result
  end

  def create_result_file(filenames)
    if @options[:result_file]
      log "Writing output filenames to #{@options[:result_file]}"
      File.open(@options[:result_file], "w") do |f|
        filenames.each do |filename|
          f.puts filename
        end
      end
    end
  end

  def create_metadata_result_file(output_dir)
    if @options[:metadata_result_file]
      log "Writing metadata to #{@options[:metadata_result_file]}"
      File.open(@options[:metadata_result_file], "w") do |f|
        f.puts "GIT_COMMIT=#{Shellwords.escape @commit}"
        f.puts "GIT_SHA=#{Shellwords.escape @sha}"
        f.puts "GIT_SUBJECT=#{Shellwords.escape @subject}"
        f.puts "OUTPUT_DIR=#{Shellwords.escape output_dir}"
      end
    end
  end

  def setup_index_symlinks
    create_symlink_to_commit_subdir(output_date_path)
    create_symlink_to_commit_subdir(output_tag_path) if @options[:create_release_symlink]
  end

  # Save a file in the commit directory that tells the user what tag this came from.
  def create_release_file
    if @options[:create_release_symlink]
      filename = "#{output_commit_subdir}/RELEASE-VERSION.TXT"
      log "Creating release file #{filename}"
      File.open(filename, "w") do |f|
        f.puts @passenger_version
      end
    end
  end

  def create_symlink_to_commit_subdir(symlink_path)
    log "Creating directory symlink #{symlink_path} to commit #{@commit}"
    make_output_subdir(File.dirname(symlink_path))
    temp_symlink_path = "#{symlink_path}.tmp#{$$}"
    begin
      ln_s output_commit_subdir, temp_symlink_path, :verbose => true
      puts "mv #{temp_symlink_path} #{symlink_path}"
      File.rename(temp_symlink_path, symlink_path)
    rescue Exception
      puts "# An error occurred, rolling back..."
      rm_f temp_symlink_path, :verbose => true
      raise
    end
  end

  def remove_build_output
    log "An error occurred. Rolling back..."
    sh "rm", "-rf", output_save_subdir
  end

  def binary_compat_id
    @binary_compat_id ||=
      if target = ENV['MACOSX_DEPLOYMENT_TARGET']
        "x86_64-macosx-#{target}"
      else
        `./bin/passenger-config --cxx-compat-id`.strip
      end
  end

  def maketar(filename, input, chdir = nil)
    args = ["tar", "-czf", "#{filename}.tmp", *input]
    args << { :chdir => chdir } if chdir
    sh(*args)
    sh("mv", "#{filename}.tmp", filename)
    filename
  end

  def nginx_url
    "http://nginx.org/download/#{nginx_basename}.tar.gz"
  end

  def nginx_tarball
    "#{@options[:repo_dir]}/#{nginx_basename}.tar.gz"
  end

  def nginx_basename
    "nginx-#{@preferred_nginx_version}"
  end

  def passenger_code_dir
    "#{@options[:repo_dir]}/main"
  end

  def output_save_subdir
    "#{@options[:output_dir]}/by_commit/#{@commit}.tmp"
  end

  def output_commit_subdir
    "#{@options[:output_dir]}/by_commit/#{@commit}"
  end

  def output_date_path
    "#{@options[:output_dir]}/by_date/#{Date.today}"
  end

  def output_tag_path
    "#{@options[:output_dir]}/by_release/#{@passenger_version}"
  end

  def inline_sed(command, filename)
    if RUBY_PLATFORM =~ /darwin/
      sh "sed", "-i", "", command, filename
    else
      sh "sed", "-i", command, filename
    end
  end

  def strip_all
    if RUBY_PLATFORM =~ /darwin/
      "strip"
    else
      "strip --strip-all"
    end
  end

  def drake
    ENV['DRAKE'] || "drake"
  end
end

Autobuild.new.start