Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement DiscoveredDependency model #486

Merged
merged 47 commits into from
Aug 31, 2022
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
47 commits
Select commit Hold shift + click to select a range
e516a97
Implement package assembly in scancode.io #447
JonoYang Aug 4, 2022
c9fbb90
Minor formatting changes for consistency #447
tdruez Aug 5, 2022
49f3e22
Create DiscoveredPackages before other models #447
JonoYang Aug 9, 2022
cd2dc2d
Revert "Create DiscoveredPackages before other models #447"
JonoYang Aug 9, 2022
63a2f60
Update migration #444
JonoYang Aug 9, 2022
06a3bf2
Return package_uids in for_packages #444
JonoYang Aug 10, 2022
bb1cd74
Create directory Resources in docker pipeline #485
JonoYang Aug 24, 2022
9466bb0
Implement package assembly in scancode.io #447
JonoYang Aug 4, 2022
14ed8a8
Implement package assembly in scancode.io #447
JonoYang Aug 4, 2022
d451e06
Create DiscoveredDependency model #447
JonoYang Aug 4, 2022
8a38284
Update fields on DiscoveredDependency #447
JonoYang Aug 5, 2022
6d89345
Properly pluralize verbose name #447
JonoYang Aug 6, 2022
8579ecd
Create new argument for create_from_data #447
JonoYang Aug 8, 2022
d7abb38
Update prefetch_related #447
JonoYang Aug 8, 2022
3bbc748
Prefetch related models in output code #447
JonoYang Aug 9, 2022
03ab20e
Import scancode.io 30.2.0 scans in load_codebase
JonoYang Aug 10, 2022
4ac4248
Revert changes for importing old scancode.io scans
JonoYang Aug 12, 2022
9749622
Regen migrations for DiscoveredDependency #447
JonoYang Aug 12, 2022
f8ee81d
Migrate DiscoveredPackage.dependencies #447
JonoYang Aug 13, 2022
5101c8b
Update test expectations #447
JonoYang Aug 15, 2022
3f90882
Remove accidentally committed files #447
JonoYang Aug 15, 2022
c61db9e
Update migration logic #447
JonoYang Aug 15, 2022
5338129
Add PackageURLMixin to DiscoveredDependency #447
JonoYang Aug 15, 2022
2b9aa39
Set DiscoveredDependencies purl fields #447
JonoYang Aug 16, 2022
18efa00
Store purl values in purl fields #447
JonoYang Aug 16, 2022
962346a
Remove purl field from DiscoveredDependency #447
JonoYang Aug 16, 2022
daf3866
Update DependencyFilterSet #447
JonoYang Aug 17, 2022
e70876b
Don't show DiscoveredDependencies purl fields #447
JonoYang Aug 19, 2022
794698b
Update package detail view dependencies tab #447
JonoYang Aug 25, 2022
bfa735d
Add package_type to dependency serializer #511
JonoYang Aug 25, 2022
2eca733
Update expected test results
JonoYang Aug 25, 2022
66fc9a0
Add dependency table column #447
JonoYang Aug 25, 2022
ccd046a
Use tabset in dependency detail view #447
JonoYang Aug 26, 2022
633a656
Update dependency list view #447
JonoYang Aug 26, 2022
e8c2298
Set DiscoveredDependency serializer fields #511
JonoYang Aug 26, 2022
483228e
Create donut chart for package type #447
JonoYang Aug 26, 2022
984a0a8
Consolidate migrations #447
JonoYang Aug 26, 2022
40862fe
Update dependency JSON ordering #447
JonoYang Aug 27, 2022
5d8cd43
Set proper discovereddependencies related_name #447
tdruez Aug 29, 2022
01a4648
Fix template indentation #447
tdruez Aug 29, 2022
dba4891
Refactor update_from_data method into a UpdateFromDataMixin #447
tdruez Aug 29, 2022
45b0e3b
Fix the ProjectSerializer fields #447
tdruez Aug 29, 2022
fa24ca1
Fix test_scanpipe_api_project_detail unit test #447
tdruez Aug 29, 2022
25fc3e9
Add HTML title for list views #506
tdruez Aug 29, 2022
87cb66b
Update dependency tabs #447
JonoYang Aug 30, 2022
0fa6783
Use UpdateFromDataMixin #447
JonoYang Aug 30, 2022
89d41be
Fix formatting #447
tdruez Aug 31, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Use UpdateFromDataMixin #447
    * Use UpdateFromDataMixin in DiscoveredDependency
    * Create test for DiscoveredDependency.update_from_data()

Signed-off-by: Jono Yang <jyang@nexb.com>
  • Loading branch information
JonoYang committed Aug 30, 2022
commit 0fa6783d451a384eb85c279a89c953f92b361d04
27 changes: 1 addition & 26 deletions scanpipe/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -1971,6 +1971,7 @@ class DiscoveredDependencyQuerySet(ProjectRelatedQuerySet):
class DiscoveredDependency(
ProjectRelatedModel,
SaveProjectErrorMixin,
UpdateFromDataMixin,
PackageURLMixin,
):
"""
Expand Down Expand Up @@ -2147,32 +2148,6 @@ def create_from_data(

return discovered_dependency

def update_from_data(self, data):
"""
Update this object instance with the provided `data`.
The `save()` is called only if at least one field was modified.
"""
model_fields = self.__class__.model_fields()
updated_fields = []

for field_name, value in data.items():
skip_reasons = [
not value,
field_name not in model_fields,
]
if any(skip_reasons):
continue

current_value = getattr(self, field_name, None)
if not current_value or current_value != value:
setattr(self, field_name, value)
updated_fields.append(field_name)

if updated_fields:
self.save()

return updated_fields


class WebhookSubscription(UUIDPKModel, ProjectRelatedModel):
target_url = models.URLField(_("Target URL"), max_length=1024)
Expand Down
15 changes: 15 additions & 0 deletions scanpipe/tests/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@
for_package_uid = "pkg:deb/debian/adduser@3.118?uuid=610bed29-ce39-40e7-92d6-fd8b"
dependency_data1 = {
"purl": "pkg:pypi/dask",
"package_type": "pypi",
"extracted_requirement": "dask<2023.0.0,>=2022.6.0",
"scope": "install",
"is_runtime": True,
Expand All @@ -103,6 +104,20 @@
"datasource_id": "pypi_sdist_pkginfo",
}

dependency_data2 = {
"purl": "pkg:gem/appraisal@2.2.0",
"package_type": "gem",
"extracted_requirement": "",
"scope": "dependencies",
"is_runtime": True,
"is_optional": False,
"is_resolved": True,
"dependency_uid": "pkg:gem/appraisal@2.2.0?uuid=1907f061-911b-4980-a2d4-ae1a9ed871a9",
"for_package_uid": for_package_uid,
"datafile_path": "data.tar.gz-extract/Gemfile.lock",
"datasource_id": "gemfile_lock",
}

license_policies = [
{
"license_key": "apache-2.0",
Expand Down
33 changes: 33 additions & 0 deletions scanpipe/tests/test_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
from scanpipe.pipes.input import copy_input
from scanpipe.pipes.input import copy_inputs
from scanpipe.tests import dependency_data1
from scanpipe.tests import dependency_data2
from scanpipe.tests import license_policies_index
from scanpipe.tests import mocked_now
from scanpipe.tests import package_data1
Expand Down Expand Up @@ -1313,6 +1314,38 @@ def test_scanpipe_model_create_user_creates_auth_token(self):
self.assertTrue(basic_user.auth_token.key)
self.assertEqual(40, len(basic_user.auth_token.key))

def test_scanpipe_discovered_dependency_model_update_from_data(self):
package = DiscoveredPackage.create_from_data(self.project1, package_data1)
resource = CodebaseResource.objects.create(
project=self.project1, path="data.tar.gz-extract/Gemfile.lock"
)
dependency = DiscoveredDependency.create_from_data(
self.project1, dependency_data2
)

new_data = {
"name": "new name",
"extracted_requirement": "new requirement",
"scope": "new scope",
"unknown_field": "value",
}
updated_fields = dependency.update_from_data(new_data)
self.assertEqual(["extracted_requirement"], updated_fields)

dependency.refresh_from_db()
# PURL field, not updated
self.assertEqual(dependency_data2["name"], dependency.name)
# Empty field, updated
self.assertEqual(
new_data["extracted_requirement"], dependency.extracted_requirement
)
# Already a value, not updated
self.assertEqual(dependency_data2["scope"], dependency.scope)

updated_fields = dependency.update_from_data(new_data, override=True)
self.assertEqual(["scope"], updated_fields)
self.assertEqual(new_data["scope"], dependency.scope)


class ScanPipeModelsTransactionTest(TransactionTestCase):
"""
Expand Down