INF-597: Terraform config files not updating and Flower docker contai…

…ner not building successfully. (#598)

observatory-platform/observatory/platform/terraform/build.sh

@@ -3,6 +3,8 @@
 # Documentation recommends to sleep for 30 seconds first:
 sleep 30
 
+echo " ----- Install Docker (using apt-get) ----- "
+
 # Install Docker
 sudo apt-get update
 sudo apt-get -y install apt-transport-https ca-certificates curl gnupg-agent software-properties-common
@@ -12,11 +14,15 @@ sudo apt-get update
 sudo apt-get -y install docker-ce docker-ce-cli containerd.io
 sudo service docker restart
 
+echo " ----- Make the airflow user and add it to the docker group ----- "
+
 # Make the airflow user and add it to the docker group
 sudo useradd --home-dir /home/airflow --shell /bin/bash --create-home airflow
 sudo usermod -aG docker airflow
 sudo newgrp docker
 
+echo " ----- Install Docker Compose v1.29.2 and Berlas v0.5.0 ----- "
+
 # Install Docker Compose
 sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-Linux-x86_64" -o /usr/local/bin/docker-compose
 sudo chmod +x /usr/local/bin/docker-compose
@@ -25,13 +31,17 @@ sudo chmod +x /usr/local/bin/docker-compose
 sudo curl -L "https://storage.googleapis.com/berglas/0.5.0/linux_amd64/berglas" -o /usr/local/bin/berglas
 sudo chmod +x /usr/local/bin/berglas
 
+echo " ----- Install Google Compute Monitoring agent ----- "
+
 # Install Google Compute Monitoring agent
 curl -sSO https://dl.google.com/cloudagents/add-monitoring-agent-repo.sh
 sudo bash add-monitoring-agent-repo.sh
 sudo apt-get update
 sudo apt-get install -y 'stackdriver-agent=6.*'
 sudo service stackdriver-agent start
 
+echo " ----- Make airflow and docker directories, move packages, and clean up files ----- "
+
 # Make directories
 sudo mkdir -p /opt/airflow/logs
 sudo mkdir /opt/airflow/dags
@@ -47,8 +57,9 @@ sudo cp -r /tmp/opt/observatory/build/docker/* /opt/observatory/build/docker
 # Remove tmp
 sudo rm -r /tmp
 
-# Own all /opt directories
+# Own all /opt directories and packer home folder
 sudo chown -R airflow /opt/
+sudo chown -R airflow /home/packer/
 
 # Set working directory and environment variables for building docker containers
 cd /opt/observatory/build/docker
@@ -59,6 +70,8 @@ export HOST_AIRFLOW_UI_PORT=8080
 export HOST_ELASTIC_PORT=9200
 export HOST_KIBANA_PORT=5601
 
+echo " ----- Building docker containers with docker-compose, running as airflow user ----- "
+
 # Pull and build Docker containers
 PRESERVE_ENV="HOST_USER_ID,HOST_REDIS_PORT,HOST_FLOWER_UI_PORT,HOST_AIRFLOW_UI_PORT,HOST_ELASTIC_PORT,HOST_KIBANA_PORT"
 sudo -u airflow --preserve-env=${PRESERVE_ENV} bash -c "docker-compose -f docker-compose.observatory.yml pull"

observatory-platform/observatory/platform/terraform/terraform_builder.py

@@ -31,7 +31,7 @@
 from observatory.platform.utils.proc_utils import stream_process
 
 
-def copy_dir(source_path: str, destination_path: str, ignore):
+def copy_dir(source_path: str, destination_path: str):
     distutils.dir_util.copy_tree(source_path, destination_path)
 
 
@@ -89,16 +89,19 @@ def make_files(self):
             shutil.rmtree(self.packages_build_path)
         os.makedirs(self.packages_build_path)
 
-        ignore = shutil.ignore_patterns("__pycache__", "*.eggs", "*.egg-info")
-
         # Copy local packages
         for package in self.config.python_packages:
             if package.type == "editable":
                 destination_path = os.path.join(self.packages_build_path, package.name)
-                copy_dir(package.host_package, destination_path, ignore)
+                copy_dir(package.host_package, destination_path)
+
+        # Clear terraform/terraform path
+        if os.path.exists(self.terraform_build_path):
+            shutil.rmtree(self.terraform_build_path)
+        os.makedirs(self.terraform_build_path)
 
-        # Copy terraform files into build/terraform: ignore jinja2 templates
-        copy_dir(self.terraform_path, self.terraform_build_path, shutil.ignore_patterns("*.jinja2", "__pycache__"))
+        # Copy terraform files into build/terraform
+        copy_dir(self.terraform_path, self.terraform_build_path)
 
         # Make startup scripts
         self.make_startup_script(True, "startup-main.tpl")