Merge branch '513-feature-request-update-module-version-archive-to-in…

…clude-only-specified-path-contents' into 'main'

Resolve "Feature Request: Update Module Version Archive to Include Only Specified Path Contents"

Closes #513

See merge request pub/terrareg!390

docs/API.md

@@ -793,6 +793,19 @@ Provide interface to create module provider.
 #### POST
 
 Handle update to settings.
+##### Arguments
+
+| Argument | Location (JSON POST body or query string argument) | Type | Required | Default | Help |
+|----------|----------------------------------------------------|------|----------|---------|------|
+| git_provider_id | json | str | False | `None` | ID of the git provider to associate to module provider. |
+| repo_base_url_template | json | str | False | `None` | Templated base git URL. |
+| repo_clone_url_template | json | str | False | `None` | Templated git clone URL. |
+| repo_browse_url_template | json | str | False | `None` | Templated URL for browsing repository. |
+| git_tag_format | json | str | False | `None` | Module provider git tag format. |
+| git_path | json | str | False | `None` | Path within git repository that the module exists. |
+| archive_git_path | json | boolean | False | `False` | Whether to generate module archives from the git_path directory. Otherwise, archives are generated from the root |
+| csrf_token | json | str | False | `None` | CSRF token |
+
 
 
 ## ApiTerraregModuleProviderDelete
@@ -817,6 +830,23 @@ Provide interface to update module provider settings.
 #### POST
 
 Handle update to settings.
+##### Arguments
+
+| Argument | Location (JSON POST body or query string argument) | Type | Required | Default | Help |
+|----------|----------------------------------------------------|------|----------|---------|------|
+| git_provider_id | json | str | False | `None` | ID of the git provider to associate to module provider. |
+| repo_base_url_template | json | str | False | `None` | Templated base git repository URL. |
+| repo_clone_url_template | json | str | False | `None` | Templated git clone URL. |
+| repo_browse_url_template | json | str | False | `None` | Templated URL for browsing repository. |
+| git_tag_format | json | str | False | `None` | Module provider git tag format. |
+| git_path | json | str | False | `None` | Path within git repository that the module exists. |
+| archive_git_path | json | boolean | False | `None` | Whether to generate module archives from the git_path directory. Otherwise, archives are generated from the root |
+| verified | json | boolean | False | `None` | Whether module provider is marked as verified. |
+| namespace | json | str | False | `None` | Name of new namespace to move module/module provider to a new namespace |
+| module | json | str | False | `None` | New name of module |
+| provider | json | str | False | `None` | New provider for module |
+| csrf_token | json | str | False | `None` | CSRF token |
+
 
 
 ## ApiTerraregModuleProviderIntegrations

terrareg/alembic/versions/f9a80ea383cc_add_archive_git_path_column_to_module_.py

@@ -0,0 +1,97 @@
+"""Add archive_git_path column to module provider
+
+Revision ID: f9a80ea383cc
+Revises: 9dbcb4240c55
+Create Date: 2024-03-14 06:05:26.867657
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'f9a80ea383cc'
+down_revision = '9dbcb4240c55'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('module_provider', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('archive_git_path', sa.Boolean(), nullable=True))
+    with op.batch_alter_table('module_version', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('archive_git_path', sa.Boolean(), nullable=True))
+        batch_op.add_column(sa.Column('git_path', sa.String(length=1024), nullable=True))
+
+    bind = op.get_bind()
+    module_provider_paths = bind.execute("""SELECT id, git_path FROM module_provider""")
+    for module_provider_id, git_path in module_provider_paths:
+        bind.execute(
+            sa.sql.text(
+                """
+                UPDATE module_version
+                SET git_path=:git_path
+                WHERE module_provider_id=:module_provider_id"""
+            ),
+            git_path=git_path,
+            module_provider_id=module_provider_id,
+        )
+
+    if op.get_bind().engine.name == 'mysql':
+        op.alter_column('audit_history', 'action',
+            existing_type=sa.Enum(
+                'NAMESPACE_CREATE', 'NAMESPACE_MODIFY_NAME', 'NAMESPACE_MODIFY_DISPLAY_NAME', 'MODULE_PROVIDER_CREATE', 'MODULE_PROVIDER_DELETE', 'MODULE_PROVIDER_UPDATE_GIT_TAG_FORMAT',
+                'MODULE_PROVIDER_UPDATE_GIT_PROVIDER', 'MODULE_PROVIDER_UPDATE_GIT_PATH', 'MODULE_PROVIDER_UPDATE_GIT_CUSTOM_BASE_URL',
+                'MODULE_PROVIDER_UPDATE_GIT_CUSTOM_CLONE_URL', 'MODULE_PROVIDER_UPDATE_GIT_CUSTOM_BROWSE_URL', 'MODULE_PROVIDER_UPDATE_VERIFIED',
+                'MODULE_VERSION_INDEX', 'MODULE_VERSION_PUBLISH', 'MODULE_VERSION_DELETE', 'USER_GROUP_CREATE', 'USER_GROUP_DELETE', 'USER_GROUP_NAMESPACE_PERMISSION_ADD',
+                'USER_GROUP_NAMESPACE_PERMISSION_MODIFY', 'USER_GROUP_NAMESPACE_PERMISSION_DELETE', 'USER_LOGIN',
+                'MODULE_PROVIDER_UPDATE_NAMESPACE', 'MODULE_PROVIDER_UPDATE_MODULE_NAME', 'MODULE_PROVIDER_UPDATE_PROVIDER_NAME', 'NAMESPACE_DELETE', 'MODULE_PROVIDER_REDIRECT_DELETE',
+                'GPG_KEY_CREATE', 'GPG_KEY_DELETE', 'PROVIDER_CREATE', 'PROVIDER_DELETE', 'PROVIDER_VERSION_INDEX', 'PROVIDER_VERSION_DELETE', 'REPOSITORY_CREATE', 'REPOSITORY_UPDATE', 'REPOSITORY_DELETE',
+                name='auditaction'),
+            type_=sa.Enum(
+                'NAMESPACE_CREATE', 'NAMESPACE_MODIFY_NAME', 'NAMESPACE_MODIFY_DISPLAY_NAME', 'MODULE_PROVIDER_CREATE', 'MODULE_PROVIDER_DELETE', 'MODULE_PROVIDER_UPDATE_GIT_TAG_FORMAT',
+                'MODULE_PROVIDER_UPDATE_GIT_PROVIDER', 'MODULE_PROVIDER_UPDATE_GIT_PATH', 'MODULE_PROVIDER_UPDATE_GIT_CUSTOM_BASE_URL',
+                'MODULE_PROVIDER_UPDATE_GIT_CUSTOM_CLONE_URL', 'MODULE_PROVIDER_UPDATE_GIT_CUSTOM_BROWSE_URL', 'MODULE_PROVIDER_UPDATE_VERIFIED',
+                'MODULE_VERSION_INDEX', 'MODULE_VERSION_PUBLISH', 'MODULE_VERSION_DELETE', 'USER_GROUP_CREATE', 'USER_GROUP_DELETE', 'USER_GROUP_NAMESPACE_PERMISSION_ADD',
+                'USER_GROUP_NAMESPACE_PERMISSION_MODIFY', 'USER_GROUP_NAMESPACE_PERMISSION_DELETE', 'USER_LOGIN',
+                'MODULE_PROVIDER_UPDATE_NAMESPACE', 'MODULE_PROVIDER_UPDATE_MODULE_NAME', 'MODULE_PROVIDER_UPDATE_PROVIDER_NAME', 'NAMESPACE_DELETE', 'MODULE_PROVIDER_REDIRECT_DELETE',
+                'GPG_KEY_CREATE', 'GPG_KEY_DELETE', 'PROVIDER_CREATE', 'PROVIDER_DELETE', 'PROVIDER_VERSION_INDEX', 'PROVIDER_VERSION_DELETE', 'REPOSITORY_CREATE', 'REPOSITORY_UPDATE', 'REPOSITORY_DELETE',
+                'MODULE_PROVIDER_UPDATE_ARCHIVE_GIT_PATH',
+                name='auditaction'),
+            nullable=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+
+    if op.get_bind().engine.name == 'mysql':
+        op.alter_column('audit_history', 'action',
+            existing_type=sa.Enum(
+                'NAMESPACE_CREATE', 'NAMESPACE_MODIFY_NAME', 'NAMESPACE_MODIFY_DISPLAY_NAME', 'MODULE_PROVIDER_CREATE', 'MODULE_PROVIDER_DELETE', 'MODULE_PROVIDER_UPDATE_GIT_TAG_FORMAT',
+                'MODULE_PROVIDER_UPDATE_GIT_PROVIDER', 'MODULE_PROVIDER_UPDATE_GIT_PATH', 'MODULE_PROVIDER_UPDATE_GIT_CUSTOM_BASE_URL',
+                'MODULE_PROVIDER_UPDATE_GIT_CUSTOM_CLONE_URL', 'MODULE_PROVIDER_UPDATE_GIT_CUSTOM_BROWSE_URL', 'MODULE_PROVIDER_UPDATE_VERIFIED',
+                'MODULE_VERSION_INDEX', 'MODULE_VERSION_PUBLISH', 'MODULE_VERSION_DELETE', 'USER_GROUP_CREATE', 'USER_GROUP_DELETE', 'USER_GROUP_NAMESPACE_PERMISSION_ADD',
+                'USER_GROUP_NAMESPACE_PERMISSION_MODIFY', 'USER_GROUP_NAMESPACE_PERMISSION_DELETE', 'USER_LOGIN',
+                'MODULE_PROVIDER_UPDATE_NAMESPACE', 'MODULE_PROVIDER_UPDATE_MODULE_NAME', 'MODULE_PROVIDER_UPDATE_PROVIDER_NAME', 'NAMESPACE_DELETE', 'MODULE_PROVIDER_REDIRECT_DELETE',
+                'GPG_KEY_CREATE', 'GPG_KEY_DELETE', 'PROVIDER_CREATE', 'PROVIDER_DELETE', 'PROVIDER_VERSION_INDEX', 'PROVIDER_VERSION_DELETE', 'REPOSITORY_CREATE', 'REPOSITORY_UPDATE', 'REPOSITORY_DELETE',
+                'MODULE_PROVIDER_UPDATE_ARCHIVE_GIT_PATH',
+                name='auditaction'),
+            type_=sa.Enum(
+                'NAMESPACE_CREATE', 'NAMESPACE_MODIFY_NAME', 'NAMESPACE_MODIFY_DISPLAY_NAME', 'MODULE_PROVIDER_CREATE', 'MODULE_PROVIDER_DELETE', 'MODULE_PROVIDER_UPDATE_GIT_TAG_FORMAT',
+                'MODULE_PROVIDER_UPDATE_GIT_PROVIDER', 'MODULE_PROVIDER_UPDATE_GIT_PATH', 'MODULE_PROVIDER_UPDATE_GIT_CUSTOM_BASE_URL',
+                'MODULE_PROVIDER_UPDATE_GIT_CUSTOM_CLONE_URL', 'MODULE_PROVIDER_UPDATE_GIT_CUSTOM_BROWSE_URL', 'MODULE_PROVIDER_UPDATE_VERIFIED',
+                'MODULE_VERSION_INDEX', 'MODULE_VERSION_PUBLISH', 'MODULE_VERSION_DELETE', 'USER_GROUP_CREATE', 'USER_GROUP_DELETE', 'USER_GROUP_NAMESPACE_PERMISSION_ADD',
+                'USER_GROUP_NAMESPACE_PERMISSION_MODIFY', 'USER_GROUP_NAMESPACE_PERMISSION_DELETE', 'USER_LOGIN',
+                'MODULE_PROVIDER_UPDATE_NAMESPACE', 'MODULE_PROVIDER_UPDATE_MODULE_NAME', 'MODULE_PROVIDER_UPDATE_PROVIDER_NAME', 'NAMESPACE_DELETE', 'MODULE_PROVIDER_REDIRECT_DELETE',
+                'GPG_KEY_CREATE', 'GPG_KEY_DELETE', 'PROVIDER_CREATE', 'PROVIDER_DELETE', 'PROVIDER_VERSION_INDEX', 'PROVIDER_VERSION_DELETE', 'REPOSITORY_CREATE', 'REPOSITORY_UPDATE', 'REPOSITORY_DELETE',
+                name='auditaction'),
+            nullable=False)
+
+    with op.batch_alter_table('module_provider', schema=None) as batch_op:
+        batch_op.drop_column('archive_git_path')
+    with op.batch_alter_table('module_version', schema=None) as batch_op:
+        batch_op.drop_column('archive_git_path')
+        batch_op.drop_column('git_path')
+    # ### end Alembic commands ###

terrareg/audit_action.py

@@ -16,6 +16,7 @@ class AuditAction(Enum):
     MODULE_PROVIDER_UPDATE_GIT_TAG_FORMAT = "module_provider_update_git_tag_format"
     MODULE_PROVIDER_UPDATE_GIT_PROVIDER = "module_provider_update_git_provider"
     MODULE_PROVIDER_UPDATE_GIT_PATH = "module_provider_update_git_path"
+    MODULE_PROVIDER_UPDATE_ARCHIVE_GIT_PATH = "module_provider_update_archive_git_path"
     MODULE_PROVIDER_UPDATE_GIT_CUSTOM_BASE_URL = "module_provider_update_git_custom_base_url"
     MODULE_PROVIDER_UPDATE_GIT_CUSTOM_CLONE_URL = "module_provider_update_git_custom_clone_url"
     MODULE_PROVIDER_UPDATE_GIT_CUSTOM_BROWSE_URL = "module_provider_update_git_custom_browse_url"

terrareg/database.py

@@ -454,6 +454,7 @@ def initialise(self):
             sqlalchemy.Column('repo_browse_url_template', sqlalchemy.String(URL_COLUMN_SIZE)),
             sqlalchemy.Column('git_tag_format', sqlalchemy.String(GENERAL_COLUMN_SIZE)),
             sqlalchemy.Column('git_path', sqlalchemy.String(URL_COLUMN_SIZE)),
+            sqlalchemy.Column('archive_git_path', sqlalchemy.Boolean, default=False),
             sqlalchemy.Column('verified', sqlalchemy.Boolean),
             sqlalchemy.Column(
                 'git_provider_id',
@@ -503,6 +504,8 @@ def initialise(self):
             ),
             sqlalchemy.Column('version', sqlalchemy.String(GENERAL_COLUMN_SIZE)),
             sqlalchemy.Column('git_sha', sqlalchemy.String(GENERAL_COLUMN_SIZE)),
+            sqlalchemy.Column('git_path', sqlalchemy.String(URL_COLUMN_SIZE)),
+            sqlalchemy.Column('archive_git_path', sqlalchemy.Boolean, default=False),
             sqlalchemy.Column(
                 'module_details_id',
                 sqlalchemy.ForeignKey(

terrareg/models.py

@@ -2413,6 +2413,11 @@ def tag_ref_regex(self):
         """Return regex match for git ref to match version"""
         return self.get_tag_regex(self.git_ref_format)
 
+    @property
+    def archive_git_path(self) -> bool:
+        """Return whether archives should only contain the contents of the git_path of the repo"""
+        return bool(self._get_db_row()['archive_git_path'])
+
     @property
     def git_path(self):
         """Return path of module within git"""
@@ -2628,6 +2633,19 @@ def get_git_provider(self):
             return GitProvider.get(id=self._get_db_row()['git_provider_id'])
         return None
 
+    def update_archive_git_path(self, archive_git_path):
+        """Set archive_git_path value"""
+        original_value = self._get_db_row()['archive_git_path']
+        if original_value != archive_git_path:
+            terrareg.audit.AuditEvent.create_audit_event(
+                action=terrareg.audit_action.AuditAction.MODULE_PROVIDER_UPDATE_ARCHIVE_GIT_PATH,
+                object_type=self.__class__.__name__,
+                object_id=self.id,
+                old_value=original_value,
+                new_value=archive_git_path
+            )
+        self.update_attributes(archive_git_path=archive_git_path)
+
     def get_git_clone_url(self):
         """Return URL to perform git clone"""
         template = None
@@ -3048,6 +3066,7 @@ def get_terrareg_api_details(self):
             "git_provider_id": git_provider.pk if git_provider else None,
             "git_tag_format": self.git_tag_format,
             "git_path": self.git_path,
+            "archive_git_path": self.archive_git_path,
             "repo_base_url_template": self._get_db_row()['repo_base_url_template'],
             "repo_clone_url_template": self._get_db_row()['repo_clone_url_template'],
             "repo_browse_url_template": self._get_db_row()['repo_browse_url_template']
@@ -3678,9 +3697,14 @@ def path(self):
         return ''
 
     @property
-    def git_path(self):
+    def git_path(self) -> Optional[str]:
         """Return path of module within git"""
-        return self._module_provider.git_path
+        return self._get_db_row()['git_path']
+
+    @property
+    def archive_git_path(self) -> bool:
+        """Return whether archives should only contain the contents of the git_path of the repo"""
+        return bool(self._get_db_row()['archive_git_path'])
 
     @property
     def id(self):
@@ -3892,7 +3916,7 @@ def get_git_clone_url(self):
 
         return None
 
-    def get_source_download_url(self, request_domain: str, direct_http_request: bool, path: Optional[bool]=None):
+    def get_source_download_url(self, request_domain: str, direct_http_request: bool, path: Optional[str]=None):
         """Return URL to download source file."""
         rendered_url = None
 
@@ -3914,12 +3938,12 @@ def get_source_download_url(self, request_domain: str, direct_http_request: bool
             # Check if git_path has been set and prepend to path, if set.
             path = os.path.join(self.git_path or '', path or '')
 
-            # Remove any trailing slashes from path
-            if path and path.endswith('/'):
-                path = path[:-1]
-
-            # Check if path is present for module (only used for submodules)
+            # Check if path is present for module
             if path:
+
+                # Remove any trailing slashes from path
+                path = path.lstrip('/').rstrip('/')
+
                 rendered_url = '{rendered_url}//{path}'.format(
                     rendered_url=rendered_url,
                     path=path)
@@ -3941,6 +3965,25 @@ def get_source_download_url(self, request_domain: str, direct_http_request: bool
         if config.ALLOW_MODULE_HOSTING is not terrareg.config.ModuleHostingMode.DISALLOW:
             url = '/v1/terrareg/modules/{0}/{1}'.format(self.id, self.archive_name_zip)
 
+            # If archive does not contain just the git_path,
+            # check if git_path has been set and prepend to path, if set.
+            if not self.archive_git_path:
+                path = os.path.join(self.git_path or '', path or '')
+
+            # Check if path is present for module
+            if path:
+                # Remove any trailing slashes from path
+                path = path.lstrip('/').rstrip('/')
+
+                rendered_url = '{rendered_url}//{path}'.format(
+                    rendered_url=rendered_url,
+                    path=path)
+            if path:
+                url = '{url}//{path}'.format(
+                    url=url,
+                    path=path
+                )
+
             # If authentication is required, generate pre-signed URL
             if not config.ALLOW_UNAUTHENTICATED_ACCESS:
                 presign_key = TerraformSourcePresignedUrl.generate_presigned_key(url=url)

terrareg/module_extractor.py

@@ -64,11 +64,22 @@ def extract_directory(self):
         """Return path of extract directory."""
         return self._extract_directory.name
 
+    @property
+    def archive_source_directory(self):
+        """Return directory that is used for generating the archives"""
+        # If the module provider is configured to only archive the git_path
+        # of the source, limit to only this path
+        if self._module_version.module_provider.archive_git_path:
+            return self.module_directory
+
+        # Otherwise, return the root of the repository
+        return self.extract_directory
+
     @property
     def module_directory(self):
         """Return path of module directory, based on configured git path."""
-        if self._module_version.git_path:
-            return safe_join_paths(self._extract_directory.name, self._module_version.git_path)
+        if self._module_version.module_provider.git_path:
+            return safe_join_paths(self._extract_directory.name, self._module_version.module_provider.git_path)
         else:
             return self._extract_directory.name
 
@@ -364,12 +375,11 @@ def tar_filter(tarinfo):
                 return None
             return tarinfo
 
-
         # Create tar.gz
         with tempfile.TemporaryDirectory(suffix='generate-archive') as temp_dir:
             tar_file_path = os.path.join(temp_dir, self._module_version.archive_name_tar_gz)
             with tarfile.open(tar_file_path, "w:gz") as tar:
-                tar.add(self.extract_directory, arcname='', recursive=True, filter=tar_filter)
+                tar.add(self.archive_source_directory, arcname='', recursive=True, filter=tar_filter)
 
             # Add tar file to file storage
             file_storage.upload_file(tar_file_path, self._module_version.base_directory, self._module_version.archive_name_tar_gz)
@@ -384,7 +394,7 @@ def tar_filter(tarinfo):
             zip_file_path = os.path.join(temp_dir, self._module_version.archive_name_zip)
             subprocess.call(
                 ['zip', '-r', zip_file_path, '--exclude=./.git/*', '.'],
-                cwd=self.extract_directory
+                cwd=self.archive_source_directory
             )
             file_storage.upload_file(zip_file_path, self._module_version.base_directory, self._module_version.archive_name_zip)
 
@@ -445,7 +455,9 @@ def _insert_database(
             published=False,
             git_sha=git_sha,
             internal=terrareg_metadata.get('internal', False),
-            extraction_version=EXTRACTION_VERSION
+            extraction_version=EXTRACTION_VERSION,
+            git_path=self._module_version.module_provider.git_path,
+            archive_git_path=self._module_version.module_provider.archive_git_path,
         )
 
     def _process_submodule(self, submodule: 'terrareg.models.BaseSubmodule'):
@@ -660,6 +672,11 @@ def _extract_description(self, readme_content):
 
     def process_upload(self):
         """Handle data extraction from module source."""
+
+        # Ensure base directory exists
+        if not os.path.isdir(self.module_directory):
+            raise PathDoesNotExistError(f"Base module could not be found (git path: {self._module_version.git_path})")
+
         # Generate the archive, unless the module has a git clone URL and
         # the config for deleting externally hosted artifacts is enabled.
         # Always perform this first before making any modifications to the repo