Skip to content

Commit

Permalink
added metranet lib download in gh actions
Browse files Browse the repository at this point in the history
  • Loading branch information
wolfidan committed Feb 26, 2024
1 parent 954f99b commit c98a7a4
Show file tree
Hide file tree
Showing 2 changed files with 43 additions and 2 deletions.
10 changes: 8 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,12 @@ jobs:

steps:
- uses: actions/checkout@v2

- name: Download metranet
run: python get_metranet_lib.py
working-directory: ${{github.workspace}}/pyart/testing/
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
# Install dependencies
- name: Setup Conda Environment
uses: mamba-org/setup-micromamba@v1
Expand Down Expand Up @@ -60,7 +65,8 @@ jobs:
shell: bash -l {0}
run: |
python -m pytest -v --cov=./ --cov-report=xml
env:
METRANETLIB_PATH: ${{github.workspace}}/pyart/testing/lib/
- name: Upload code coverage to Codecov
uses: codecov/codecov-action@v2.1.0
with:
Expand Down
35 changes: 35 additions & 0 deletions pyart/testing/get_metranet_lib.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
import boto3
import os

BUCKET='pyart'
ENDPOINT_URL='https://eu-central-1.linodeobjects.com'
def download_metranet_lib():
linode_obj_config = {
"aws_access_key_id": os.environ['AWS_ACCESS_KEY_ID'],
"endpoint_url": ENDPOINT_URL,
'aws_secret_access_key': os.environ['AWS_SECRET_ACCESS_KEY']}

client = boto3.client("s3", **linode_obj_config)

current_directory = os.path.dirname(os.path.abspath(__file__))
lib_dir = os.path.join(current_directory, 'lib')
if not os.path.exists(lib_dir):
os.makedirs(lib_dir)

# List objects in the bucket
paginator = client.get_paginator('list_objects_v2')
for result in paginator.paginate(Bucket=BUCKET):
if 'Contents' in result:
for obj in result['Contents']:
# Get the object key
object_key = obj['Key']
# Extract file name from object key
file_name = os.path.basename(object_key)
# Construct local file path
local_file_path = os.path.join(lib_dir, file_name)
# Download the object
client.download_file(BUCKET, object_key, local_file_path)
print(f"Downloaded {object_key} to {local_file_path}")

if __name__ == '__main__':
download_metranet_lib()

0 comments on commit c98a7a4

Please sign in to comment.