Skip to content
This repository has been archived by the owner on Aug 2, 2022. It is now read-only.

Workflow Consistency / Simplification Updates in All Workflows - Part3 #336

Merged
merged 2 commits into from
Aug 4, 2020
Merged

Workflow Consistency / Simplification Updates in All Workflows - Part3 #336

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c3a3c00
Simplification workflow for docker and integTests
peterzhuamazon Aug 4, 2020
9e49b88
tweak tar parameters
peterzhuamazon Aug 4, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 72 additions & 8 deletions .github/scripts/setup_runners_service.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,21 @@
# About: Setup ES/KIBANA for integTests on *NIX based ODFE distros w/wo Security
#
# Usage: ./setup_runners_service.sh $SETUP_DISTRO $SETUP_ACTION
# $SETUP_DISTRO: zip | deb | rpm (required)
# $SETUP_DISTRO: zip | docker | deb | rpm (required)
# $SETUP_ACTION: --es | --es-nosec | --kibana | --kibana-nosec (required)
#
# Requirements: This script assumes java 14 is already installed on the servers
#
# Starting Date: 2020-07-27
# Modified Date: 2020-08-02
# Modified Date: 2020-08-04
###############################################################################################

# This script allows users to manually assign parameters
if [ "$#" -ne 2 ] || [ "$1" = "--help" ] || [ "$1" = "-h" ]
then
echo "Please assign 2 parameters when running this script"
echo "Example: $0 \$SETUP_DISTRO \$SETUP_ACTION"
echo "Example: $0 \"zip | deb | rpm\" \"--es | --es-nosec | --kibana | --kibana-nosec\""
echo "Example: $0 \"zip | docker | deb | rpm\" \"--es | --es-nosec | --kibana | --kibana-nosec\""
exit 1
fi

Expand All @@ -43,13 +43,18 @@ ES_ROOT="${ROOT}/odfe-testing/${ES_PACKAGE_NAME}"
KIBANA_PACKAGE_NAME="opendistroforelasticsearch-kibana"
KIBANA_ROOT="${ROOT}/kibana-testing/${KIBANA_PACKAGE_NAME}"

DOCKER_NAME="Test-Docker-${OD_VERSION}"
DOCKER_NAME_KIBANA="Test-Docker-Kibana-${OD_VERSION}"
DOCKER_NAME_NoSec="Test-Docker-${OD_VERSION}-NoSec"
DOCKER_NAME_KIBANA_NoSec="Test-Docker-Kibana-${OD_VERSION}-NoSec"

S3_BUCKET="artifacts.opendistroforelasticsearch.amazon.com"

#####################################################################################################

echo "############################################################"
echo "Setup ES/KIBANA to start with YES/NO security configurations"
echo "User enters $SETUP_ACTION"
echo "User enters ${SETUP_DISTRO}:${SETUP_ACTION}"
echo "############################################################"

echo "setup parameters"
Expand All @@ -63,7 +68,14 @@ if [ "$SETUP_DISTRO" = "zip" ]
then
mkdir -p $ES_ROOT
aws s3 cp s3://$S3_BUCKET/downloads/tarball/opendistro-elasticsearch/$ES_PACKAGE_NAME.tar.gz . --quiet; echo $?
tar -zxf $ES_PACKAGE_NAME.tar.gz -C $ES_ROOT --strip 1
tar -zxf $ES_PACKAGE_NAME.tar.gz -C $ES_ROOT --strip-components 1
fi

if [ "$SETUP_DISTRO" = "docker" ]
then
echo "setup docker"
echo -n > Dockerfile
echo -n > Dockerfile.kibana
fi

if [ "$SETUP_DISTRO" = "deb" ]
Expand Down Expand Up @@ -93,10 +105,20 @@ echo "setup es"
# everything needs es
if [ "$SETUP_DISTRO" = "zip" ]
then
sed -i /install_demo_configuration/d $ES_ROOT/opendistro-tar-install.sh
sed -i /^node.max_local_storage_nodes/d $ES_ROOT/config/elasticsearch.yml
mkdir -p snapshots
echo "path.repo: [\"$PWD/snapshots\"]" >> $ES_ROOT/config/elasticsearch.yml
elif [ "$SETUP_DISTRO" = "docker" ]
then
echo "FROM opendistroforelasticsearch/opendistroforelasticsearch:$OD_VERSION" >> Dockerfile
echo "RUN echo '' >> /usr/share/elasticsearch/config/elasticsearch.yml" >> Dockerfile
echo "RUN echo \"path.repo: [\\\"/usr/share/elasticsearch\\\"]\" >> /usr/share/elasticsearch/config/elasticsearch.yml" >> Dockerfile
docker build -t odfe-http:security -f Dockerfile .
sleep 5
docker run -d -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" --name $DOCKER_NAME odfe-http:security
sleep 30
echo "Temp Solution to remove the wrong configuration. need be fixed in building stage"
docker exec -t $DOCKER_NAME /bin/bash -c "sed -i /^node.max_local_storage_nodes/d /usr/share/elasticsearch/config/elasticsearch.yml"
docker stop $DOCKER_NAME
else
sudo mkdir -p /home/repo
sudo chmod 777 /home/repo
Expand All @@ -111,6 +133,14 @@ then
then
cd $ES_ROOT
nohup ./opendistro-tar-install.sh > /dev/null 2>&1 &
sleep 30
kill -9 `ps -ef | grep [e]lasticsearch | awk '{print $2}'`
sed -i /^node.max_local_storage_nodes/d ./config/elasticsearch.yml
nohup ./opendistro-tar-install.sh > /dev/null 2>&1 &
elif [ "$SETUP_DISTRO" = "docker" ]
then
docker restart $DOCKER_NAME
docker ps
else
sudo systemctl restart elasticsearch.service
fi
Expand All @@ -132,8 +162,14 @@ then

if [ "$SETUP_DISTRO" = "zip" ]
then
sed -i /install_demo_configuration/d $ES_ROOT/opendistro-tar-install.sh
$ES_ROOT/bin/elasticsearch-plugin remove opendistro_security
sed -i '/http\.port/s/^# *//' $ES_ROOT/config/elasticsearch.yml
elif [ "$SETUP_DISTRO" = "docker" ]
then
echo "RUN /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security" >> Dockerfile
docker build -t odfe-http:no-security -f Dockerfile .
sleep 5
else
sudo /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security
sudo sed -i '/http\.port/s/^# *//' /etc/elasticsearch/elasticsearch.yml
Expand All @@ -149,6 +185,10 @@ then
then
cd $ES_ROOT
nohup ./opendistro-tar-install.sh > /dev/null 2>&1 &
elif [ "$SETUP_DISTRO" = "docker" ]
then
docker run -d -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" --name $DOCKER_NAME_NoSec odfe-http:no-security
docker ps
else
sudo systemctl restart elasticsearch.service
fi
Expand All @@ -172,7 +212,14 @@ then
then
mkdir -p $KIBANA_ROOT
aws s3 cp s3://$S3_BUCKET/downloads/tarball/$KIBANA_PACKAGE_NAME/$KIBANA_PACKAGE_NAME-$OD_VERSION.tar.gz . --quiet; echo $?
tar -zxf $KIBANA_PACKAGE_NAME-$OD_VERSION.tar.gz -C $KIBANA_ROOT --strip 1
tar -zxf $KIBANA_PACKAGE_NAME-$OD_VERSION.tar.gz -C $KIBANA_ROOT --strip-components 1
elif [ "$SETUP_DISTRO" = "docker" ]
then
echo "FROM opendistroforelasticsearch/opendistroforelasticsearch-kibana:$OD_VERSION" >> Dockerfile.kibana
echo "RUN sed -i /^opendistro_security/d /usr/share/kibana/config/kibana.yml" >> Dockerfile.kibana
echo "RUN sed -i 's/https/http/' /usr/share/kibana/config/kibana.yml" >> Dockerfile.kibana
docker build -t odfe-kibana-http:security -f Dockerfile.kibana .
sleep 5
else
sudo apt install $KIBANA_PACKAGE_NAME=$OD_VERSION* -y || sudo yum install $KIBANA_PACKAGE_NAME-$OD_VERSION -y
fi
Expand All @@ -184,8 +231,17 @@ then
then
cd $ES_ROOT
nohup ./opendistro-tar-install.sh > /dev/null 2>&1 &
sleep 30
kill -9 `ps -ef | grep [e]lasticsearch | awk '{print $2}'`
sed -i /^node.max_local_storage_nodes/d ./config/elasticsearch.yml
nohup ./opendistro-tar-install.sh > /dev/null 2>&1 &
cd $KIBANA_ROOT
nohup ./bin/kibana > /dev/null 2>&1 &
elif [ "$SETUP_DISTRO" = "docker" ]
then
docker restart $DOCKER_NAME
docker run -d -p 5601:5601 --name $DOCKER_NAME_KIBANA --network="host" odfe-kibana-http:security
docker ps
else
sudo systemctl restart elasticsearch.service
sudo systemctl restart kibana.service
Expand Down Expand Up @@ -218,6 +274,14 @@ then
nohup ./opendistro-tar-install.sh > /dev/null 2>&1 &
cd $KIBANA_ROOT
nohup ./bin/kibana > /dev/null 2>&1 &
elif [ "$SETUP_DISTRO" = "docker" ]
then
echo "RUN /usr/share/kibana/bin/kibana-plugin remove opendistro_security" >> Dockerfile.kibana
docker build -t odfe-kibana-http:no-security -f Dockerfile.kibana .
sleep 5
docker run -d -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" --name $DOCKER_NAME_NoSec odfe-http:no-security
docker run -d -p 5601:5601 --name $DOCKER_NAME_KIBANA_NoSec --network="host" odfe-kibana-http:no-security
docker ps
else
sudo /usr/share/kibana/bin/kibana-plugin remove opendistro_security --allow-root
sudo sed -i /^opendistro_security/d /etc/kibana/kibana.yml
Expand Down
Loading