Download GitHub release artifacts

omego/artifacts.py

@@ -8,6 +8,7 @@
 from builtins import str
 from past.builtins import basestring
 from builtins import object
+import json
 import os
 import logging
 
@@ -51,7 +52,9 @@ def __init__(self, args):
             args.ci = 'https://ci.openmicroscopy.org'
         if not args.branch:
             args.branch = 'latest'
-        if args.build or re.match(r'[A-Za-z]\w+-\w+', args.branch):
+        if args.github:
+            self.artifacts = GithubArtifacts(args)
+        elif args.build or re.match(r'[A-Za-z]\w+-\w+', args.branch):
             self.artifacts = JenkinsArtifacts(args)
         elif re.match('[0-9]+|latest$', args.branch):
             self.artifacts = ReleaseArtifacts(args)
@@ -472,6 +475,56 @@ def read_downloads(dlurl):
         return dl_icever
 
 
+class GithubArtifacts(ArtifactsList):
+    """
+    Fetch artifacts from GitHub releases
+    """
+
+    def __init__(self, args):
+        super(GithubArtifacts, self).__init__()
+        self.args = args
+
+        if re.match(r'[0-9]+\.[0-9]+\.[0-9]+', args.branch):
+            dl_url = 'https://api.github.com/repos/%s/releases/tags/v%s' % (
+                args.github, args.branch)
+        else:
+            raise ArtifactException(
+                'Only GitHub tags are supported', args.branch)
+
+        if args.ice:
+            raise ArtifactException(
+                'Ice argument not supported for GitHub releases', args.ice)
+
+        artifacturls = self.read_downloads(dl_url)
+        if len(artifacturls) <= 0:
+            raise AttributeError(
+                "No artifacts, please check the GitHub releases page.")
+        self.find_artifacts(artifacturls)
+
+    @staticmethod
+    def read_downloads(dlurl):
+        url = None
+        d = None
+        try:
+            url = fileutils.open_url(dlurl)
+            log.debug('Fetching html from %s code:%d', url.url, url.code)
+            if url.code != 200:
+                log.error('Failed to get HTML from %s (code %d)',
+                          url.url, url.code)
+                raise Stop(
+                    20, 'Downloads page failed, is the version correct?')
+
+            d = json.load(url)
+        except HTTPError as e:
+            log.error('Failed to get HTML from %s (%s)', dlurl, e)
+            raise Stop(20, 'Downloads page failed, is the version correct?')
+        finally:
+            if url:
+                url.close()
+
+        return [a['browser_download_url'] for a in d['assets']]
+
+
 class DownloadCommand(Command):
     """
     Download an OMERO artifact from either a downloads or a Continuous

omego/env.py

@@ -131,6 +131,11 @@ def __init__(self, parser):
             " DOWNLOADURL/omero/<version>/artifacts. Default: "
             "http://downloads.openmicroscopy.org")
 
+        Add(group, "github", "",
+            help="GitHub repository containing release artifacts in form "
+            "'org/repository', if set this will override all other artifact "
+            "sources, default disabled")
+
         Add(group, "ice",
             "", help="Ice version, default is the latest (release only)")
 

test/integration/test_download.py

@@ -88,12 +88,6 @@ def testDownloadSym(self, tmpdir):
             assert sym2 == (old_div(tmpdir, 'custom.sym'))
             assert sym2.isdir()
 
-    def testDownloadRelease(self, tmpdir):
-        with tmpdir.as_cwd():
-            self.download('--release', 'latest', '--ice', self.ice)
-            files = tmpdir.listdir()
-            assert len(files) == 2
-
     def testDownloadNonExistingArtifact(self):
         with pytest.raises(AttributeError):
             self.download('-n', '--release', '5.3', '--ice', '3.3')
@@ -108,6 +102,40 @@ def testDownloadBuildNumber(self):
         assert 'No artifacts' in exc.value.args[0]
 
 
+class TestDownloadRelease(Downloader):
+
+    def setup_class(self):
+        # python artifact no longer exists
+        self.artifact = 'apidoc'
+
+    def testDownloadRelease(self, tmpdir):
+        with tmpdir.as_cwd():
+            self.download('--release', 'latest', '--ice', '3.6')
+            files = tmpdir.listdir()
+            assert len(files) == 2
+
+
+class TestDownloadGithub(Downloader):
+
+    def setup_class(self):
+        self.artifact = 'insight'
+
+    def testDownloadGithub(self, tmpdir):
+        with tmpdir.as_cwd():
+            self.download(
+                '--release', '5.5.8',
+                '--github', 'ome/omero-insight',
+                '--sym', 'auto')
+        files = tmpdir.listdir()
+        assert len(files) == 3
+        print([f.basename for f in files])
+        assert sorted(f.basename for f in files) == [
+            'OMERO.insight',
+            'OMERO.insight-5.5.8',
+            'OMERO.insight-5.5.8.zip',
+        ]
+
+
 class TestDownloadBioFormats(Downloader):
 
     def setup_class(self):

test/unit/test_artifacts.py

@@ -22,11 +22,17 @@
 from builtins import str
 from builtins import object
 import pytest
+import json
 from mox3 import mox
 
 from yaclifw.framework import Stop
 from omego.artifacts import ArtifactException, ArtifactsList
-from omego.artifacts import Artifacts, JenkinsArtifacts, ReleaseArtifacts
+from omego.artifacts import (
+    Artifacts,
+    GithubArtifacts,
+    JenkinsArtifacts,
+    ReleaseArtifacts,
+)
 # Import whatever XML module was imported in omego.artifacts to avoid dealing
 # with different versions
 from omego.artifacts import XML
@@ -153,6 +159,23 @@ def close(self):
         pass
 
 
+class MockGithubUrl(object):
+    pageurl = 'https://api.github.com/repos/ome/example/releases/tags/v0.0.0'
+    artifact_url = ('https://github.com/ome/example/releases/download/v0.0.0/'
+                    'OMERO.example-0.0.0.zip')
+    page_dict = {'assets': [{'browser_download_url': artifact_url}]}
+
+    def __init__(self):
+        self.code = 200
+        self.url = self.pageurl
+
+    def read(self):
+        return json.dumps(self.page_dict)
+
+    def close(self):
+        pass
+
+
 class Args(object):
     def __init__(self, matrix):
         if matrix:
@@ -173,6 +196,7 @@ def __init__(self, matrix):
         self.branch = 'TEST-build'
         self.downloadurl = MockDownloadUrl.downloadurl
         self.sym = None
+        self.github = None
 
 
 class MoxBase(object):
@@ -305,6 +329,31 @@ def test_read_downloads(self):
         self.mox.VerifyAll()
 
 
+class TestGithubArtifacts(MoxBase):
+
+    def test_init(self):
+        self.mox.StubOutWithMock(fileutils, 'open_url')
+        fileutils.open_url(MockGithubUrl.pageurl).AndReturn(
+            MockGithubUrl())
+        self.mox.ReplayAll()
+
+        args = Args(False)
+        args.branch = '0.0.0'
+        args.github = 'ome/example'
+        a = GithubArtifacts(args)
+        assert a.get('example') == MockGithubUrl.artifact_url
+        self.mox.VerifyAll()
+
+    def test_nottag(self):
+        args = Args(False)
+        args.branch = 'latest'
+        args.github = 'ome/example'
+        with pytest.raises(ArtifactException) as exc:
+            GithubArtifacts(args)
+        assert exc.value.args[0] == 'Only GitHub tags are supported'
+        self.mox.VerifyAll()
+
+
 class TestArtifacts(MoxBase):
 
     class MockArtifacts(Artifacts):