Proof of Concept: Macaroons

requirements/main.in

@@ -28,6 +28,7 @@ paginate>=0.5.2
 paginate_sqlalchemy
 passlib>=1.6.4
 psycopg2
+pymacaroons
 pyramid>=1.9a2
 pyramid_jinja2>=2.5
 pyramid_mailer>=0.14.1

requirements/main.txt

@@ -369,6 +369,41 @@ pycparser==2.18 \
 Pygments==2.2.0 \
     --hash=sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d \
     --hash=sha256:dbae1046def0efb574852fab9e90209b23f556367b5a320c0bcb871c77c3e8cc
+pymacaroons==0.13.0 \
+    --hash=sha256:1e6bba42a5f66c245adf38a5a4006a99dcc06a0703786ea636098667d42903b8 \
+    --hash=sha256:3e14dff6a262fdbf1a15e769ce635a8aea72e6f8f91e408f9a97166c53b91907
+PyNaCl==1.2.1 \
+    --hash=sha256:04e30e5bdeeb2d5b34107f28cd2f5bbfdc6c616f3be88fc6f53582ff1669eeca \
+    --hash=sha256:0bfa0d94d2be6874e40f896e0a67e290749151e7de767c5aefbad1121cad7512 \
+    --hash=sha256:11aa4e141b2456ce5cecc19c130e970793fa3a2c2e6fbb8ad65b28f35aa9e6b6 \
+    --hash=sha256:13bdc1fe084ff9ac7653ae5a924cae03bf4bb07c6667c9eb5b6eb3c570220776 \
+    --hash=sha256:14339dc233e7a9dda80a3800e64e7ff89d0878ba23360eea24f1af1b13772cac \
+    --hash=sha256:1d33e775fab3f383167afb20b9927aaf4961b953d76eeb271a5703a6d756b65b \
+    --hash=sha256:2a42b2399d0428619e58dac7734838102d35f6dcdee149e0088823629bf99fbb \
+    --hash=sha256:2dce05ac8b3c37b9e2f65eab56c544885607394753e9613fd159d5e2045c2d98 \
+    --hash=sha256:63cfccdc6217edcaa48369191ae4dca0c390af3c74f23c619e954973035948cd \
+    --hash=sha256:6453b0dae593163ffc6db6f9c9c1597d35c650598e2c39c0590d1757207a1ac2 \
+    --hash=sha256:73a5a96fb5fbf2215beee2353a128d382dbca83f5341f0d3c750877a236569ef \
+    --hash=sha256:8abb4ef79161a5f58848b30ab6fb98d8c466da21fdd65558ce1d7afc02c70b5f \
+    --hash=sha256:8ac1167195b32a8755de06efd5b2d2fe76fc864517dab66aaf65662cc59e1988 \
+    --hash=sha256:8f505f42f659012794414fa57c498404e64db78f1d98dfd40e318c569f3c783b \
+    --hash=sha256:9c8a06556918ee8e3ab48c65574f318f5a0a4d31437fc135da7ee9d4f9080415 \
+    --hash=sha256:a1e25fc5650cf64f01c9e435033e53a4aca9de30eb9929d099f3bb078e18f8f2 \
+    --hash=sha256:be71cd5fce04061e1f3d39597f93619c80cdd3558a6c9ba99a546f144a8d8101 \
+    --hash=sha256:c5b1a7a680218dee9da0f1b5e24072c46b3c275d35712bc1d505b85bb03441c0 \
+    --hash=sha256:cb785db1a9468841a1265c9215c60fe5d7af2fb1b209e3316a152704607fc582 \
+    --hash=sha256:cf6877124ae6a0698404e169b3ba534542cfbc43f939d46b927d956daf0a373a \
+    --hash=sha256:d0eb5b2795b7ee2cbcfcadacbe95a13afbda048a262bd369da9904fecb568975 \
+    --hash=sha256:d3a934e2b9f20abac009d5b6951067cfb5486889cb913192b4d8288b216842f1 \
+    --hash=sha256:d795f506bcc9463efb5ebb0f65ed77921dcc9e0a50499dedd89f208445de9ecb \
+    --hash=sha256:d8aaf7e5d6b0e0ef7d6dbf7abeb75085713d0100b4eb1a4e4e857de76d77ac45 \
+    --hash=sha256:de2aaca8386cf4d70f1796352f2346f48ddb0bed61dc43a3ce773ba12e064031 \
+    --hash=sha256:e0d38fa0a75f65f556fb912f2c6790d1fa29b7dd27a1d9cc5591b281321eaaa9 \
+    --hash=sha256:eb2acabbd487a46b38540a819ef67e477a674481f84a82a7ba2234b9ba46f752 \
+    --hash=sha256:eeee629828d0eb4f6d98ac41e9a3a6461d114d1d0aa111a8931c049359298da0 \
+    --hash=sha256:f5836463a3c0cca300295b229b6c7003c415a9d11f8f9288ddbd728e2746524c \
+    --hash=sha256:f5ce9e26d25eb0b2d96f3ef0ad70e1d3ae89b5d60255c462252a3e456a48c053 \
+    --hash=sha256:fabf73d5d0286f9e078774f3435601d2735c94ce9e514ac4fb945701edead7e4
 pyparsing==2.2.0 \
     --hash=sha256:fee43f17a9c4087e7ed1605bd6df994c6173c1e977d7ade7b651292fab2bd010 \
     --hash=sha256:0832bcf47acd283788593e7a0f542407bd9550a55a8a8435214a1960e04bcb04 \

warehouse/accounts/__init__.py

@@ -21,6 +21,10 @@
     BasicAuthAuthenticationPolicy,
     SessionAuthenticationPolicy,
 )
+from warehouse.macaroons.auth_policy import (
+    MacaroonAuthenticationPolicy,
+    MacaroonAuthorizationPolicy,
+)
 from warehouse.rate_limiting import RateLimit, IRateLimiter
 
 
@@ -78,11 +82,14 @@ def includeme(config):
         MultiAuthenticationPolicy(
             [
                 SessionAuthenticationPolicy(callback=_authenticate),
+                MacaroonAuthenticationPolicy(callback=_authenticate),
                 BasicAuthAuthenticationPolicy(check=_login),
             ]
         )
     )
-    config.set_authorization_policy(ACLAuthorizationPolicy())
+    config.set_authorization_policy(
+        MacaroonAuthorizationPolicy(policy=ACLAuthorizationPolicy())
+    )
 
     # Add a request method which will allow people to access the user object.
     config.add_request_method(_user, name="user", reify=True)

warehouse/config.py

@@ -404,6 +404,9 @@ def configure(settings=None):
     # Register our authentication support.
     config.include(".accounts")
 
+    # Register support for Macaroon based authentication
+    config.include(".macaroons")
+
     # Register logged-in views
     config.include(".manage")
 

warehouse/macaroons/__init__.py

@@ -0,0 +1,18 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from warehouse.macaroons.interfaces import IMacaroonService
+from warehouse.macaroons.services import database_macaroon_factory
+
+
+def includeme(config):
+    config.register_service_factory(database_macaroon_factory, IMacaroonService)

warehouse/macaroons/auth_policy.py

@@ -0,0 +1,120 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pyramid.authentication import CallbackAuthenticationPolicy
+from pyramid.interfaces import IAuthenticationPolicy, IAuthorizationPolicy
+from pyramid.security import Denied
+from pyramid.threadlocal import get_current_request
+from zope.interface import implementer
+
+from warehouse.cache.http import add_vary_callback
+from warehouse.macaroons.interfaces import IMacaroonService
+from warehouse.macaroons.services import InvalidMacaroon
+
+
+def extract_http_macaroon(request):
+    """
+    A helper function for the extraction of HTTP Macaroon from a given request.
+    Returns either a ``None`` if no macaroon could be found, or the byte string
+    that represents our Macaroon.
+    """
+    authorization = request.headers.get("Authorization")
+    if not authorization:
+        return None
+
+    try:
+        auth_method, auth = authorization.split(" ", 1)
+    except ValueError:
+        return None
+
+    if auth_method.lower() != "macaroon":
+        return None
+
+    return auth
+
+
+@implementer(IAuthenticationPolicy)
+class MacaroonAuthenticationPolicy(CallbackAuthenticationPolicy):
+    def __init__(self, callback=None):
+        self.callback = callback
+
+    def unauthenticated_userid(self, request):
+        # If we're calling into this API on a request, then we want to register
+        # a callback which will ensure that the response varies based on the
+        # Authorization header.
+        request.add_response_callback(add_vary_callback("Authorization"))
+
+        # We need to extract our Macaroon from the request.
+        macaroon = extract_http_macaroon(request)
+
+        # Check to see if our Macaroon exists in the database, and if so
+        # fetch the user that is associated with it.
+        macaroon_service = request.find_service(IMacaroonService, context=None)
+        return macaroon_service.find_userid(macaroon)
+
+    def remember(self, request, userid, **kw):
+        # This is a NO-OP because our Macaroon header policy doesn't allow
+        # the ability for authentication to "remember" the user id. This
+        # assumes it has been configured in clients somewhere out of band.
+        return []
+
+    def forget(self, request):
+        # This is a NO-OP because our Macaroon header policy doesn't allow
+        # the ability for authentication to "forget" the user id. This
+        # assumes it has been configured in clients somewhere out of band.
+        return []
+
+
+@implementer(IAuthorizationPolicy)
+class MacaroonAuthorizationPolicy:
+    def __init__(self, policy):
+        self.policy = policy
+
+    def permits(self, context, principals, permission):
+        # The Pyramid API doesn't let us access the request here, so we have to pull it
+        # out of the thread local instead.
+        # TODO: Work with Pyramid devs to figure out if there is a better way to support
+        #       the worklow we are using hereor not.
+        request = get_current_request()
+
+        # Our request could possibly be a None, if there isn't an active request, in
+        # that case we're going to always deny, because without a request, we can't
+        # determine if this request is authorized or not.
+        if request is None:
+            return Denied("There was no active request.")
+
+        # Re-extract our Macaroon from the request, it sucks to have to do this work
+        # twice, but I believe it is inevitable unless we pass the Macaroon back as
+        # a principal-- which doesn't seem to be the right fit for it.
+        macaroon = extract_http_macaroon(request)
+
+        # This logic will only happen on requests that are being authenticated with
+        # Macaroons. Any other request will just fall back to the standard Authorization
+        # policy.
+        if macaroon is not None:
+            macaroon_service = request.find_service(IMacaroonService, context=None)
+
+            try:
+                macaroon_service.verify(macaroon)
+            except InvalidMacaroon as exc:
+                return Denied(f"The supplied token was invalid: {str(exc)!r}")
+
+        # If our Macaroon is verified, then we'll pass this request to our underlying
+        # Authorization policy, so it can handle it's own authorization logic on
+        # the prinicpal.
+        return self.policy.permits(context, principals, permission)
+
+    def principals_allowed_by_permission(self, context, permission):
+        # We just dispatch this, because Macaroons don't restrict what principals are
+        # allowed by a particular permission, they just restrict specific requests
+        # to not have that permission.
+        return self.policy.principals_allowed_by_permission(context, permission)

warehouse/macaroons/interfaces.py

@@ -0,0 +1,25 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from zope.interface import Interface
+
+
+class IMacaroonService(Interface):
+    def find_userid(macaroon):
+        """
+        Return the id of the user associated with the given macaroon.
+        """
+
+    def verify(macaroon):
+        """
+        Verify the given macaroon.
+        """

warehouse/macaroons/models.py

@@ -0,0 +1,57 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import pymacaroons
+
+from sqlalchemy import Column, ForeignKey, DateTime, String, LargeBinary
+from sqlalchemy import orm, sql
+from sqlalchemy.dialects.postgresql import JSONB, UUID
+
+from warehouse import db
+from warehouse.accounts.models import User
+
+
+def _generate_key():
+    return os.urandom(32)
+
+
+class Macaroon(db.Model):
+
+    __tablename__ = "macaroons"
+
+    # All of our Macaroons belong to a specific user, because a caveat-less
+    # Macaroon should act the same as their password does, instead of as a
+    # global permission to upload files.
+    user_id = Column(UUID(as_uuid=True), ForeignKey("accounts_user.id"), nullable=False)
+    user = orm.relationship(User)
+
+    # Store some information about the Macaroon to give users some mechanism
+    # to differentiate between them.
+    description = Column(String(100), nullable=False, server_default="")
+    created = Column(DateTime, nullable=False, server_default=sql.func.now())
+    last_used = Column(DateTime, nullable=True)
+
+    # We'll store the caveats that were added to the Macaroon during generation
+    # to allow users to see in their management UI what the total possible
+    # scope of their macaroon is.
+    caveats = Column(JSONB, nullable=False, server_default=sql.text("'{}'"))
+
+    # It might be better to move this default into the database, that way we
+    # make it less likely that something does it incorrectly (since the
+    # default would be to generate a random key). However, it appears the
+    # PostgreSQL pgcrypto extension uses OpenSSL RAND_bytes if available
+    # instead of urandom. This is less than optimal, and we would generally
+    # prefer to just always use usrandom. Thus we'll do this ourselves here
+    # in our application.
+    key = Column(LargeBinary, nullable=False, default=_generate_key)

warehouse/macaroons/services.py

@@ -0,0 +1,70 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pymacaroons
+
+from sqlalchemy.orm import joinedload
+from sqlalchemy.orm.exc import NoResultFound
+from zope.interface import implementer
+
+from warehouse.macaroons.interfaces import IMacaroonService
+from warehouse.macaroons.models import Macaroon as Macaroon
+
+
+class InvalidMacaroon(Exception):
+    ...
+
+
+@implementer(IMacaroonService)
+class DatabaseMacaroonService:
+    def __init__(self, db_session):
+        self.db = db_session
+
+    def _get_record_from_db(self, macaroon):
+        # TODO: Better handle parsing the identifier here.
+        macaroon_id = macaroon.identifier.split()[1].split(b":")[1].decode("utf8")
+
+        try:
+            dm = (
+                self.db.query(Macaroon)
+                .options(joinedload("user"))
+                .filter(Macaroon.id == macaroon_id)
+                .one()
+            )
+        except NoResultFound:
+            return
+
+        return dm
+
+    def find_userid(self, macaroon):
+        m = pymacaroons.Macaroon.deserialize(macaroon)
+        dm = self._get_record_from_db(m)
+
+        return None if dm is None else dm.user.id
+
+    def verify(self, macaroon):
+        m = pymacaroons.Macaroon.deserialize(macaroon)
+        dm = self._get_record_from_db(m)
+
+        if dm is None:
+            raise InvalidMacaroon
+
+        verifier = pymacaroons.Verifier()
+
+        if not verifier.verify(m, dm.key):
+            raise InvalidMacaroon
+        else:
+            return True
+
+
+def database_macaroon_factory(context, request):
+    return DatabaseMacaroonService(request.db)