Skip to content

Commit

Permalink
Add build metadata to boxes and build artifacts.
Browse files Browse the repository at this point in the history 
For each Packer template that is run via `bin/bento`, a JSON file of
build metadata will be written to the `builds/` directory, which looks
like the following (using the `ubuntu-14.100i386` template as an example):

    {
      "name": "ubuntu-14.10-i386",
      "version": "2.0.20150528211301",
      "build_timestamp": "20150528211301",
      "git_revision": "6b23dd8d8ff0fb9cc4473f510bc3c54f0b415d1b",
      "box_basename": "chef__ubuntu-14.10-i386-2.0.20150528211301.git.6b23dd8d8ff0fb9cc4473f510bc3c54f0b415d1b",
      "atlas_org": "chef",
      "arch": "32",
      "template": "ubuntu-14.10-i386",
      "md5": {
        "chef__ubuntu-14.10-i386-2.0.20150528211301.git.6b23dd8d8ff0fb9cc4473f510bc3c54f0b415d1b.parallels.box": "e3a18b096cddc73384f0912c3a65ebad",
        "chef__ubuntu-14.10-i386-2.0.20150528211301.git.6b23dd8d8ff0fb9cc4473f510bc3c54f0b415d1b.virtualbox.box": "106f2ca4e6da18663e7216a72dd62e56",
        "chef__ubuntu-14.10-i386-2.0.20150528211301.git.6b23dd8d8ff0fb9cc4473f510bc3c54f0b415d1b.vmware.box": "8990550bc2a0e2e7515ed3433ec54b46"
      },
      "sha256": {
        "chef__ubuntu-14.10-i386-2.0.20150528211301.git.6b23dd8d8ff0fb9cc4473f510bc3c54f0b415d1b.parallels.box": "0a0e3c9369de005a456f0cd7d94ba4d4b562d7231c11d9c5af8e40ef77131d3d",
        "chef__ubuntu-14.10-i386-2.0.20150528211301.git.6b23dd8d8ff0fb9cc4473f510bc3c54f0b415d1b.virtualbox.box": "0c23480a99294aea8f42daea2576a41820ec3bebb99a9d0a8ab72a3de1b24137",
        "chef__ubuntu-14.10-i386-2.0.20150528211301.git.6b23dd8d8ff0fb9cc4473f510bc3c54f0b415d1b.vmware.box": "9128b66ef4bae323a123fcdd0be5a598bb538f822295ab6bf043e7630a49b608"
      }
    }

In addition to the "sidecar" metadata file, a trimmed down version will
is added to each Vagrant box in `/etc/bento-metadata.json`. Using the
example above, here is what the file would look like:

    {
      "name": "ubuntu-14.10-i386",
      "version": "2.0.20150528211301",
      "build_timestamp": "20150528211301",
      "git_revision": "6b23dd8d8ff0fb9cc4473f510bc3c54f0b415d1b_dirty",
      "box_basename": "chef__ubuntu-14.10-i386-2.0.20150528211301.git.6b23dd8d8ff0fb9cc4473f510bc3c54f0b415d1b_dirty",
      "atlas_org": "chef",
      "arch": "32",
      "template": "ubuntu-14.10-i386"
    }

Also note that this changes the file naming scheme of the resulting box
artifacts in an effort to host multiple builds of the same templates in
one directory while maintaining enough information about the box within
the filename itself.

Using the same example as above, the VirtualBox provider box name is:

    chef__ubuntu-14.10-i386-2.0.20150528211301.git.6b23dd8d8ff0fb9cc4473f510bc3c54f0b415d1b.virtualbox.box

Which uses the following recipe to construct the filename:

* `atlas_org` value (default: `"chef"`)
* double underscore, which could be later interpreted as a slash (`/`)
  for an Atalas box name
* `name` value which may or may not equal the name of the template
  (captured as the `template` value)
* a dash
* `version` value, which removes the last digit in a version string and
  replaces it with the `build_timestamp` (a
  Year/Month/Day/Hour/Minute/Second format in UTC timezone)
* a period
* the string `"git"`
* a period
* `git_revision` value, which will append `"_dirty"` if the current
  state of the git repository is not completely clean (i.e., there are
  uncommitted changes which happens in active development)
* a period
* the value of the `{{.Provider}}` Packer variable, being one of
  `"virtualbox"`, `"vmware"`, or `"parallels"`
* finished with `".box"`

Closes #364
  • Loading branch information
@fnichol
fnichol committed May 29, 2015
1 parent 61290b3 commit 529bc75
Show file tree
Hide file tree
Showing 51 changed files with 923 additions and 108 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,4 @@ packer.log
.DS_Store
/packer-*/
*.variables.json
/builds/
161 changes: 153 additions & 8 deletions bin/bento
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@ $stderr.sync = true

require "benchmark"
require "digest"
require "json"
require "optparse"
require "ostruct"
require "tempfile"

class Options

Expand Down Expand Up @@ -133,44 +135,187 @@ class BuildRunner

include Common

attr_reader :templates, :dry_run, :debug, :builds
attr_reader :templates, :dry_run, :debug, :builds, :build_timestamp

def initialize(opts)
@templates = opts.templates
@dry_run = opts.dry_run
@debug = opts.debug
@builds = opts.builds
@build_timestamp = Time.now.gmtime.strftime("%Y%m%d%H%M%S")
end

def start
banner("Starting build for templates: #{templates}")
time = Benchmark.measure do
templates.each { |template| packer(template) }
templates.each { |template| build_template(template) }
end
banner("Build finished in #{duration(time.real)}.")
end

def packer(template)
cmd = packer_cmd(template)
def build_template(template)
Tempfile.open("#{template}-metadata.json") do |md_file|
Tempfile.open("#{template}-metadata-var-file") do |var_file|
write_box_metadata(template, md_file)
write_var_file(template, md_file.path, var_file)
packer(template, var_file.path)
write_final_metadata(template)
end
end
end

def packer(template, var_file)
cmd = packer_cmd(template, var_file)
banner("[#{template}] Running: '#{cmd.join(' ')}'")
time = Benchmark.measure do
system(*cmd) or raise "[#{template}] Error building, exited #{$?}"
end
banner("[#{template}] Finished in #{duration(time.real)}.")
end

def packer_cmd(template)
def packer_cmd(template, var_file)
vars = "#{template}.variables.json"
cmd = %W[packer build #{template}.json]
cmd = %W[packer build -var-file=#{var_file} #{template}.json]
cmd.insert(2, "-var-file=#{vars}") if File.exist?(vars)
cmd.insert(2, "-only=#{builds}") if builds
cmd.insert(2, "-debug") if debug
cmd.insert(0, "echo") if dry_run
cmd
end

def git_sha
%x{git rev-parse --short HEAD}.strip
def write_box_metadata(template, io)
md = BuildMetadata.new(template, build_timestamp).read

io.write(JSON.pretty_generate(md))
io.close
end

def write_final_metadata(template)
md = BuildMetadata.new(template, build_timestamp).read
path = File.join(File.dirname(__FILE__), "..", "builds")
filename = File.join(path, "#{md[:box_basename]}.metadata.json")
checksums = ChecksumMetadata.new(path, md[:box_basename]).read

md[:md5] = checksums[:md5]
md[:sha256] = checksums[:sha256]

File.open(filename, "wb") { |file| file.write(JSON.pretty_generate(md)) }
end

def write_var_file(template, md_file, io)
md = BuildMetadata.new(template, build_timestamp).read

io.write(JSON.pretty_generate({
box_basename: md[:box_basename],
build_timestamp: md[:build_timestamp],
git_revision: md[:git_revision],
metadata: md_file,
version: md[:version]
}))
io.close
end
end

class ChecksumMetadata

def initialize(path, box_basename)
@base = File.join(path, box_basename)
end

def read
{
md5: md5_checksums,
sha256: sha256_checksums
}
end

private

attr_reader :base

def md5_checksums
Hash[Dir.glob("#{base}.*.box").map { |box|
[File.basename(box), Digest::MD5.file(box).hexdigest]
}]
end

def sha256_checksums
Hash[Dir.glob("#{base}.*.box").map { |box|
[File.basename(box), Digest::SHA256.file(box).hexdigest]
}]
end
end

class BuildMetadata

def initialize(template, build_timestamp)
@template = template
@build_timestamp = build_timestamp
end

def read
{
name: name,
version: version,
build_timestamp: build_timestamp,
git_revision: git_revision,
box_basename: box_basename,
atlas_org: atlas_org,
arch: template_vars.fetch("arch", UNKNOWN),
template: template_vars.fetch("template", UNKNOWN),
}
end

private

UNKNOWN = "__unknown__".freeze
DEFAULT_ATLAS_ORG = "chef".freeze

attr_reader :template, :build_timestamp

def atlas_org
merged_vars.fetch("atlas_org", DEFAULT_ATLAS_ORG)
end

def box_basename
"#{atlas_org}__#{name}-#{version}.git.#{git_revision}"
end

def git_revision
sha = %x{git rev-parse HEAD}.strip

git_clean? ? sha : "#{sha}_dirty"
end

def git_clean?
%{git status --porcelain}.strip.empty?
end

def merged_vars
@merged_vars ||= begin
if File.exist?("#{template}.variables.json")
template_vars.merge(JSON.load(IO.read("#{template}.variables.json")))
else
template_vars
end
end
end

def name
merged_vars.fetch("name", template)
end

def template_vars
@template_vars ||= JSON.load(IO.read("#{template}.json")).fetch("variables")
end

def user_prefix
merged_vars.fetch("user_prefix", DEFAULT_USER_PREFIX)
end

def version
merged_vars.fetch("version", "#{UNKNOWN}.TIMESTAMP").
rpartition(".").first.concat(".#{build_timestamp}")
end
end

Expand Down
18 changes: 16 additions & 2 deletions centos-5.11-i386.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,15 +102,21 @@
],
"post-processors": [
{
"output": "builds/{{.Provider}}/opscode_centos-5.11-i386.box",
"output": "builds/{{user `box_basename`}}.{{.Provider}}.box",
"type": "vagrant"
}
],
"provisioners": [
{
"destination": "/tmp/bento-metadata.json",
"source": "{{user `metadata`}}",
"type": "file"
},
{
"environment_vars": [],
"execute_command": "echo 'vagrant' | {{.Vars}} sudo -S -E bash '{{.Path}}'",
"scripts": [
"scripts/common/metadata.sh",
"scripts/common/vagrant.sh",
"scripts/common/sshd.sh",
"scripts/common/vmtools.sh",
Expand All @@ -121,7 +127,15 @@
}
],
"variables": {
"mirror": "http://mirrors.kernel.org/centos"
"arch": "32",
"box_basename": "centos-5.11-i386",
"build_timestamp": "{{isotime \"20060102150405\"}}",
"git_revision": "__unknown_git_revision__",
"metadata": "floppy/dummy_metadata.json",
"mirror": "http://mirrors.kernel.org/centos",
"name": "centos-5.11-i386",
"template": "centos-5.11-i386",
"version": "2.0.TIMESTAMP"
}
}

18 changes: 16 additions & 2 deletions centos-5.11-x86_64.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,15 +102,21 @@
],
"post-processors": [
{
"output": "builds/{{.Provider}}/opscode_centos-5.11.box",
"output": "builds/{{user `box_basename`}}.{{.Provider}}.box",
"type": "vagrant"
}
],
"provisioners": [
{
"destination": "/tmp/bento-metadata.json",
"source": "{{user `metadata`}}",
"type": "file"
},
{
"environment_vars": [],
"execute_command": "echo 'vagrant' | {{.Vars}} sudo -S -E bash '{{.Path}}'",
"scripts": [
"scripts/common/metadata.sh",
"scripts/common/vagrant.sh",
"scripts/common/sshd.sh",
"scripts/common/vmtools.sh",
Expand All @@ -121,7 +127,15 @@
}
],
"variables": {
"mirror": "http://mirrors.kernel.org/centos"
"arch": "64",
"box_basename": "centos-5.11",
"build_timestamp": "{{isotime \"20060102150405\"}}",
"git_revision": "__unknown_git_revision__",
"metadata": "floppy/dummy_metadata.json",
"mirror": "http://mirrors.kernel.org/centos",
"name": "centos-5.11",
"template": "centos-5.11-x86_64",
"version": "2.0.TIMESTAMP"
}
}

18 changes: 16 additions & 2 deletions centos-6.6-i386.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,15 +102,21 @@
],
"post-processors": [
{
"output": "builds/{{.Provider}}/opscode_centos-6.6-i386.box",
"output": "builds/{{user `box_basename`}}.{{.Provider}}.box",
"type": "vagrant"
}
],
"provisioners": [
{
"destination": "/tmp/bento-metadata.json",
"source": "{{user `metadata`}}",
"type": "file"
},
{
"environment_vars": [],
"execute_command": "echo 'vagrant' | {{.Vars}} sudo -S -E bash '{{.Path}}'",
"scripts": [
"scripts/common/metadata.sh",
"scripts/centos/fix-slow-dns.sh",
"scripts/common/sshd.sh",
"scripts/common/vagrant.sh",
Expand All @@ -122,7 +128,15 @@
}
],
"variables": {
"mirror": "http://mirrors.kernel.org/centos"
"arch": "32",
"box_basename": "centos-6.6-i386",
"build_timestamp": "{{isotime \"20060102150405\"}}",
"git_revision": "__unknown_git_revision__",
"metadata": "floppy/dummy_metadata.json",
"mirror": "http://mirrors.kernel.org/centos",
"name": "centos-6.6-i386",
"template": "centos-6.6-i386",
"version": "2.0.TIMESTAMP"
}
}

18 changes: 16 additions & 2 deletions centos-6.6-x86_64.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,15 +102,21 @@
],
"post-processors": [
{
"output": "builds/{{.Provider}}/opscode_centos-6.6.box",
"output": "builds/{{user `box_basename`}}.{{.Provider}}.box",
"type": "vagrant"
}
],
"provisioners": [
{
"destination": "/tmp/bento-metadata.json",
"source": "{{user `metadata`}}",
"type": "file"
},
{
"environment_vars": [],
"execute_command": "echo 'vagrant' | {{.Vars}} sudo -S -E bash '{{.Path}}'",
"scripts": [
"scripts/common/metadata.sh",
"scripts/centos/fix-slow-dns.sh",
"scripts/common/sshd.sh",
"scripts/common/vagrant.sh",
Expand All @@ -122,7 +128,15 @@
}
],
"variables": {
"mirror": "http://mirrors.kernel.org/centos"
"arch": "64",
"box_basename": "centos-6.6",
"build_timestamp": "{{isotime \"20060102150405\"}}",
"git_revision": "__unknown_git_revision__",
"metadata": "floppy/dummy_metadata.json",
"mirror": "http://mirrors.kernel.org/centos",
"name": "centos-6.6",
"template": "centos-6.6-x86_64",
"version": "2.0.TIMESTAMP"
}
}

Loading

0 comments on commit 529bc75

Please sign in to comment.