From a693e5fe9cd5824e5b29ffe95743c2c7bc58ea69 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 7 Oct 2024 23:48:24 +0530 Subject: [PATCH 01/16] Add pipeline to publish scan to federatedcode - Addon pipeline to push package scan to federatedcode Signed-off-by: Keshav Priyadarshi --- scancodeio/settings.py | 7 + .../pipelines/publish_to_federatedcode.py | 97 ++++++++++++++ scanpipe/pipes/federatedcode.py | 123 ++++++++++++++++++ setup.cfg | 3 + 4 files changed, 230 insertions(+) create mode 100644 scanpipe/pipelines/publish_to_federatedcode.py create mode 100644 scanpipe/pipes/federatedcode.py diff --git a/scancodeio/settings.py b/scancodeio/settings.py index caa14fd1e..66b4c9f0e 100644 --- a/scancodeio/settings.py +++ b/scancodeio/settings.py @@ -418,3 +418,10 @@ MATCHCODEIO_USER = env.str("MATCHCODEIO_USER", default="") MATCHCODEIO_PASSWORD = env.str("MATCHCODEIO_PASSWORD", default="") MATCHCODEIO_API_KEY = env.str("MATCHCODEIO_API_KEY", default="") + +# FederatedCode integration + +FEDERATEDCODE_GIT_ACCOUNT = env.str("FEDERATEDCODE_GIT_ACCOUNT", default="") +FEDERATEDCODE_GIT_SERVICE_TOKEN = env.str("FEDERATEDCODE_GIT_SERVICE_TOKEN", default="") +FEDERATEDCODE_GIT_SERVICE_NAME = env.str("FEDERATEDCODE_GIT_SERVICE_NAME", default="") +FEDERATEDCODE_GIT_SERVICE_EMAIL = env.str("FEDERATEDCODE_GIT_SERVICE_EMAIL", default="") diff --git a/scanpipe/pipelines/publish_to_federatedcode.py b/scanpipe/pipelines/publish_to_federatedcode.py new file mode 100644 index 000000000..79d19a218 --- /dev/null +++ b/scanpipe/pipelines/publish_to_federatedcode.py @@ -0,0 +1,97 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +import shutil + +from scanpipe.pipelines import Pipeline +from scanpipe.pipes import federatedcode + + +class PublishToFederatedCode(Pipeline): + """Publish package scan to FederatedCode Git repository.""" + + download_inputs = False + is_addon = True + + @classmethod + def steps(cls): + return ( + cls.get_package, + cls.get_package_repository, + cls.clone_repository, + cls.add_scan_result, + cls.commit_and_push_changes, + cls.delete_local_clone, + ) + + def get_package(self): + has_single_package_scan = any( + run.pipeline_name == "scan_single_package" + for run in self.project.runs.all() + if run.task_exitcode == 0 + ) + + if not has_single_package_scan: + raise Exception("Run ``scan_single_package`` pipeline to get package scan.") + + if not self.project.discoveredpackages.count() == 1: + raise Exception("Scan should be for single package.") + + if not self.project.discoveredpackages.first().version: + raise Exception("Scan package is missing version.") + + configured, error = federatedcode.is_configured() + if not configured: + raise Exception(error) + + self.package = self.project.discoveredpackages.first() + + def get_package_repository(self): + self.package_scan_file, self.package_git_repo = ( + federatedcode.get_package_repository(package=self.package, logger=self.log) + ) + + def clone_repository(self): + """Clone repository to local_path.""" + self.repo = federatedcode.clone_repository( + package_repo_url=self.package_git_repo, + logger=self.log, + ) + + def add_scan_result(self): + self.relative_file_path = federatedcode.add_scan_result( + project=self.project, + repo=self.repo, + package_scan_file=self.package_scan_file, + logger=self.log, + ) + + def commit_and_push_changes(self): + federatedcode.commit_and_push_changes( + repo=self.repo, + file_to_commit=str(self.relative_file_path), + purl=self.package.purl, + logger=self.log, + ) + + def delete_local_clone(self): + shutil.rmtree(self.repo.working_dir) diff --git a/scanpipe/pipes/federatedcode.py b/scanpipe/pipes/federatedcode.py new file mode 100644 index 000000000..345bed87f --- /dev/null +++ b/scanpipe/pipes/federatedcode.py @@ -0,0 +1,123 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + + +import tempfile +import textwrap +from pathlib import Path +from urllib.parse import urljoin + +from git import Repo + +from aboutcode import hashid +from scancodeio import VERSION +from scancodeio import settings +from scanpipe.pipes.output import JSONResultsGenerator + + +def is_configured(): + """Return True if the required FederatedCode settings have been set.""" + missing_vars = [] + if not settings.FEDERATEDCODE_GIT_ACCOUNT: + missing_vars.append("FEDERATEDCODE_GIT_ACCOUNT") + if not settings.FEDERATEDCODE_GIT_SERVICE_TOKEN: + missing_vars.append("FEDERATEDCODE_GIT_SERVICE_TOKEN") + if not settings.FEDERATEDCODE_GIT_SERVICE_NAME: + missing_vars.append("FEDERATEDCODE_GIT_SERVICE_NAME") + if not settings.FEDERATEDCODE_GIT_SERVICE_EMAIL: + missing_vars.append("FEDERATEDCODE_GIT_SERVICE_EMAIL") + + if missing_vars: + return False, f'Missing environment variables: {", ".join(missing_vars)}' + + return True, "" + + +def clone_repository(package_repo_url, logger=None): + """Clone repository to local_path.""" + local_dir = tempfile.mkdtemp() + + authenticated_repo_url = package_repo_url.replace( + "https://", + f"https://{settings.FEDERATEDCODE_GIT_SERVICE_TOKEN}@", + ) + repo = Repo.clone_from(url=authenticated_repo_url, to_path=local_dir, depth=1) + + repo.config_writer(config_level="repository").set_value( + "user", "name", settings.FEDERATEDCODE_GIT_SERVICE_NAME + ).release() + + repo.config_writer(config_level="repository").set_value( + "user", "email", settings.FEDERATEDCODE_GIT_SERVICE_EMAIL + ).release() + + return repo + + +def get_package_repository(package, logger=None): + package_base_dir = hashid.get_package_base_dir(purl=package.purl) + package_repo_name = package_base_dir.parts[0] + + package_scan_file = package_base_dir / package.version / "scan.json" + package_git_repo_url = urljoin( + settings.FEDERATEDCODE_GIT_ACCOUNT, f"{package_repo_name}.git" + ) + + return package_scan_file, package_git_repo_url + + +def add_scan_result(project, repo, package_scan_file, logger=None): + relative_scan_file_path = Path(*package_scan_file.parts[1:]) + + write_to = Path(repo.working_dir) / relative_scan_file_path + + write_to.parent.mkdir(parents=True, exist_ok=True) + results_generator = JSONResultsGenerator(project) + with open(write_to, encoding="utf-8", mode="w") as file: + for chunk in results_generator: + file.write(chunk) + + return relative_scan_file_path + + +def commit_and_push_changes(repo, file_to_commit, purl, logger=None): + author_name = settings.FEDERATEDCODE_GIT_SERVICE_NAME + author_email = settings.FEDERATEDCODE_GIT_SERVICE_EMAIL + add_or_update = "Update" + + if file_to_commit in repo.untracked_files: + add_or_update = "Add" + + commit_message = f"""\ + {add_or_update} scan result for {purl} + + Tool: pkg:github/aboutcode-org/scancode.io@v{VERSION} + Reference: https://{settings.ALLOWED_HOSTS[0]}/ + + Signed-off-by: {author_name} <{author_email}> + """ + + default_branch = repo.active_branch.name + + repo.index.add([file_to_commit]) + repo.index.commit(textwrap.dedent(commit_message).strip()) + repo.git.push("origin", default_branch, "--no-verify") diff --git a/setup.cfg b/setup.cfg index f4a993dfb..ce09ab621 100644 --- a/setup.cfg +++ b/setup.cfg @@ -105,6 +105,8 @@ install_requires = bleach==6.1.0 # Antivirus clamd==1.0.2 + # FederatedCode + aboutcode.hashid>=0.1.0 [options.extras_require] dev = @@ -146,6 +148,7 @@ scancodeio_pipelines = map_deploy_to_develop = scanpipe.pipelines.deploy_to_develop:DeployToDevelop match_to_matchcode = scanpipe.pipelines.match_to_matchcode:MatchToMatchCode populate_purldb = scanpipe.pipelines.populate_purldb:PopulatePurlDB + publish_to_federatedcode = scanpipe.pipelines.publish_to_federatedcode:PublishToFederatedCode resolve_dependencies = scanpipe.pipelines.resolve_dependencies:ResolveDependencies scan_codebase = scanpipe.pipelines.scan_codebase:ScanCodebase scan_for_virus = scanpipe.pipelines.scan_for_virus:ScanForVirus From 2e2c4766cf77d1ae54b099c7b6ef0cbf81e4b5aa Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Tue, 8 Oct 2024 15:45:38 +0530 Subject: [PATCH 02/16] Add docstrings to pipeline steps Signed-off-by: Keshav Priyadarshi --- .../pipelines/publish_to_federatedcode.py | 10 +++++-- scanpipe/pipes/federatedcode.py | 26 ++++++++++--------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/scanpipe/pipelines/publish_to_federatedcode.py b/scanpipe/pipelines/publish_to_federatedcode.py index 79d19a218..3719f0162 100644 --- a/scanpipe/pipelines/publish_to_federatedcode.py +++ b/scanpipe/pipelines/publish_to_federatedcode.py @@ -44,6 +44,7 @@ def steps(cls): ) def get_package(self): + """Get the package associated with the scan.""" has_single_package_scan = any( run.pipeline_name == "scan_single_package" for run in self.project.runs.all() @@ -66,18 +67,20 @@ def get_package(self): self.package = self.project.discoveredpackages.first() def get_package_repository(self): - self.package_scan_file, self.package_git_repo = ( + """Get the Git repository URL and scan path for a given package.""" + self.package_git_repo, self.package_scan_file = ( federatedcode.get_package_repository(package=self.package, logger=self.log) ) def clone_repository(self): """Clone repository to local_path.""" self.repo = federatedcode.clone_repository( - package_repo_url=self.package_git_repo, + repo_url=self.package_git_repo, logger=self.log, ) def add_scan_result(self): + """Add package scan result to the local Git repository.""" self.relative_file_path = federatedcode.add_scan_result( project=self.project, repo=self.repo, @@ -86,12 +89,15 @@ def add_scan_result(self): ) def commit_and_push_changes(self): + """Commit and push changes to remote repository.""" federatedcode.commit_and_push_changes( repo=self.repo, file_to_commit=str(self.relative_file_path), purl=self.package.purl, logger=self.log, ) + self.log(f"Scan for '{self.package.purl}' pushed to '{self.package_git_repo}'") def delete_local_clone(self): + """Remove local clone.""" shutil.rmtree(self.repo.working_dir) diff --git a/scanpipe/pipes/federatedcode.py b/scanpipe/pipes/federatedcode.py index 345bed87f..fe6710c36 100644 --- a/scanpipe/pipes/federatedcode.py +++ b/scanpipe/pipes/federatedcode.py @@ -52,11 +52,11 @@ def is_configured(): return True, "" -def clone_repository(package_repo_url, logger=None): +def clone_repository(repo_url, logger=None): """Clone repository to local_path.""" local_dir = tempfile.mkdtemp() - authenticated_repo_url = package_repo_url.replace( + authenticated_repo_url = repo_url.replace( "https://", f"https://{settings.FEDERATEDCODE_GIT_SERVICE_TOKEN}@", ) @@ -74,18 +74,20 @@ def clone_repository(package_repo_url, logger=None): def get_package_repository(package, logger=None): + """Return the Git repository URL and scan path for a given package.""" package_base_dir = hashid.get_package_base_dir(purl=package.purl) package_repo_name = package_base_dir.parts[0] - package_scan_file = package_base_dir / package.version / "scan.json" + package_scan_path = package_base_dir / package.version / "scan.json" package_git_repo_url = urljoin( settings.FEDERATEDCODE_GIT_ACCOUNT, f"{package_repo_name}.git" ) - return package_scan_file, package_git_repo_url + return package_git_repo_url, package_scan_path def add_scan_result(project, repo, package_scan_file, logger=None): + """Add package scan result to the local Git repository.""" relative_scan_file_path = Path(*package_scan_file.parts[1:]) write_to = Path(repo.working_dir) / relative_scan_file_path @@ -99,16 +101,16 @@ def add_scan_result(project, repo, package_scan_file, logger=None): return relative_scan_file_path -def commit_and_push_changes(repo, file_to_commit, purl, logger=None): +def commit_and_push_changes( + repo, file_to_commit, purl, remote_name="origin", logger=None +): + """Commit and push changes to remote repository.""" author_name = settings.FEDERATEDCODE_GIT_SERVICE_NAME author_email = settings.FEDERATEDCODE_GIT_SERVICE_EMAIL - add_or_update = "Update" - - if file_to_commit in repo.untracked_files: - add_or_update = "Add" + change_type = "Add" if file_to_commit in repo.untracked_files else "Update" commit_message = f"""\ - {add_or_update} scan result for {purl} + {change_type} scan result for {purl} Tool: pkg:github/aboutcode-org/scancode.io@v{VERSION} Reference: https://{settings.ALLOWED_HOSTS[0]}/ @@ -119,5 +121,5 @@ def commit_and_push_changes(repo, file_to_commit, purl, logger=None): default_branch = repo.active_branch.name repo.index.add([file_to_commit]) - repo.index.commit(textwrap.dedent(commit_message).strip()) - repo.git.push("origin", default_branch, "--no-verify") + repo.index.commit(textwrap.dedent(commit_message)) + repo.git.push(remote_name, default_branch) From b51cffa2a241f8c78357ddcc674de530bb370246 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 11 Oct 2024 18:21:08 +0530 Subject: [PATCH 03/16] Store scan result in scancodeio.json file Signed-off-by: Keshav Priyadarshi --- scanpipe/pipes/federatedcode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanpipe/pipes/federatedcode.py b/scanpipe/pipes/federatedcode.py index fe6710c36..11154975a 100644 --- a/scanpipe/pipes/federatedcode.py +++ b/scanpipe/pipes/federatedcode.py @@ -78,7 +78,7 @@ def get_package_repository(package, logger=None): package_base_dir = hashid.get_package_base_dir(purl=package.purl) package_repo_name = package_base_dir.parts[0] - package_scan_path = package_base_dir / package.version / "scan.json" + package_scan_path = package_base_dir / package.version / "scancodeio.json" package_git_repo_url = urljoin( settings.FEDERATEDCODE_GIT_ACCOUNT, f"{package_repo_name}.git" ) From 930a5cc6c226d2649869dc608f3c69f44646e20a Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Tue, 15 Oct 2024 13:43:42 +0530 Subject: [PATCH 04/16] Add test for federatedcode Signed-off-by: Keshav Priyadarshi --- .../pipelines/publish_to_federatedcode.py | 3 +- scanpipe/pipes/federatedcode.py | 33 +++++---- scanpipe/tests/pipes/test_federatedcode.py | 74 +++++++++++++++++++ 3 files changed, 95 insertions(+), 15 deletions(-) create mode 100644 scanpipe/tests/pipes/test_federatedcode.py diff --git a/scanpipe/pipelines/publish_to_federatedcode.py b/scanpipe/pipelines/publish_to_federatedcode.py index 3719f0162..49b703539 100644 --- a/scanpipe/pipelines/publish_to_federatedcode.py +++ b/scanpipe/pipelines/publish_to_federatedcode.py @@ -20,7 +20,6 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. -import shutil from scanpipe.pipelines import Pipeline from scanpipe.pipes import federatedcode @@ -100,4 +99,4 @@ def commit_and_push_changes(self): def delete_local_clone(self): """Remove local clone.""" - shutil.rmtree(self.repo.working_dir) + federatedcode.delete_local_clone(repo=self.repo) diff --git a/scanpipe/pipes/federatedcode.py b/scanpipe/pipes/federatedcode.py index 11154975a..b4e658b46 100644 --- a/scanpipe/pipes/federatedcode.py +++ b/scanpipe/pipes/federatedcode.py @@ -21,6 +21,7 @@ # Visit https://github.com/aboutcode-org/scancode.io for support and download. +import shutil import tempfile import textwrap from pathlib import Path @@ -52,6 +53,20 @@ def is_configured(): return True, "" +def get_package_repository(package, logger=None): + """Return the Git repository URL and scan path for a given package.""" + FEDERATEDCODE_GIT_ACCOUNT_URL = f'{settings.FEDERATEDCODE_GIT_ACCOUNT.rstrip("/")}/' + package_base_dir = hashid.get_package_base_dir(purl=package.purl) + package_repo_name = package_base_dir.parts[0] + + package_scan_path = package_base_dir / package.version / "scancodeio.json" + package_git_repo_url = urljoin( + FEDERATEDCODE_GIT_ACCOUNT_URL, f"{package_repo_name}.git" + ) + + return package_git_repo_url, package_scan_path + + def clone_repository(repo_url, logger=None): """Clone repository to local_path.""" local_dir = tempfile.mkdtemp() @@ -73,19 +88,6 @@ def clone_repository(repo_url, logger=None): return repo -def get_package_repository(package, logger=None): - """Return the Git repository URL and scan path for a given package.""" - package_base_dir = hashid.get_package_base_dir(purl=package.purl) - package_repo_name = package_base_dir.parts[0] - - package_scan_path = package_base_dir / package.version / "scancodeio.json" - package_git_repo_url = urljoin( - settings.FEDERATEDCODE_GIT_ACCOUNT, f"{package_repo_name}.git" - ) - - return package_git_repo_url, package_scan_path - - def add_scan_result(project, repo, package_scan_file, logger=None): """Add package scan result to the local Git repository.""" relative_scan_file_path = Path(*package_scan_file.parts[1:]) @@ -123,3 +125,8 @@ def commit_and_push_changes( repo.index.add([file_to_commit]) repo.index.commit(textwrap.dedent(commit_message)) repo.git.push(remote_name, default_branch) + + +def delete_local_clone(repo): + """Remove local clone.""" + shutil.rmtree(repo.working_dir) diff --git a/scanpipe/tests/pipes/test_federatedcode.py b/scanpipe/tests/pipes/test_federatedcode.py new file mode 100644 index 000000000..6348300bf --- /dev/null +++ b/scanpipe/tests/pipes/test_federatedcode.py @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + +import shutil +import tempfile +from pathlib import Path +from unittest.mock import patch + +from django.test import TestCase + +import git + +from scanpipe import models +from scanpipe.pipes import federatedcode +from scanpipe.tests import make_package + + +class ScanPipeFederatedCodeTest(TestCase): + def setUp(self): + self.project1 = models.Project.objects.create(name="Analysis") + + @patch( + "scanpipe.pipes.federatedcode.settings.FEDERATEDCODE_GIT_ACCOUNT", + "https://github.com/test/", + ) + def test_scanpipe_pipes_federatedcode_get_package_repository(self): + package = make_package( + project=self.project1, + package_url="pkg:npm/foobar@v1.2.3", + version="v.1.2.3", + ) + expected_git_repo = "https://github.com/test/aboutcode-packages-03f1.git" + expected_scan_path = "aboutcode-packages-03f1/npm/foobar/v1.2.3/scancodeio.json" + git_repo, scan_path = federatedcode.get_package_repository(package=package) + + self.assertEqual(expected_git_repo, git_repo) + self.assertEqual(expected_scan_path, str(scan_path)) + + def test_scanpipe_pipes_federatedcode_add_scan_result(self): + local_dir = tempfile.mkdtemp() + repo = git.Repo.init(local_dir) + + federatedcode.add_scan_result( + self.project1, repo, Path("repo/npm/foobar/v1.2.3/scancodeio.json") + ) + + self.assertIn("npm/foobar/v1.2.3/scancodeio.json", repo.untracked_files) + shutil.rmtree(repo.working_dir) + + def test_scancpipe_pipes_federatedcode_delete_local_clone(self): + local_dir = tempfile.mkdtemp() + repo = git.Repo.init(local_dir) + federatedcode.delete_local_clone(repo) + + self.assertEqual(False, Path(local_dir).exists()) From fabb3086e53f718797e01bd2316b1255afb56484 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 23 Oct 2024 13:53:18 +0530 Subject: [PATCH 05/16] Add project_purl field to project Signed-off-by: Keshav Priyadarshi --- scanpipe/api/serializers.py | 1 + scanpipe/forms.py | 15 +++++++++++++++ .../migrations/0068_project_project_purl.py | 18 ++++++++++++++++++ scanpipe/models.py | 6 ++++++ .../templates/scanpipe/project_settings.html | 8 ++++++++ 5 files changed, 48 insertions(+) create mode 100644 scanpipe/migrations/0068_project_project_purl.py diff --git a/scanpipe/api/serializers.py b/scanpipe/api/serializers.py index 5da4f1186..32c3a3001 100644 --- a/scanpipe/api/serializers.py +++ b/scanpipe/api/serializers.py @@ -203,6 +203,7 @@ class Meta: "name", "url", "uuid", + "project_purl", "upload_file", "upload_file_tag", "input_urls", diff --git a/scanpipe/forms.py b/scanpipe/forms.py index f854235aa..ffbb01828 100644 --- a/scanpipe/forms.py +++ b/scanpipe/forms.py @@ -25,6 +25,7 @@ from django.core.exceptions import ObjectDoesNotExist from django.core.exceptions import ValidationError +from packageurl import PackageURL from taggit.forms import TagField from taggit.forms import TagWidget @@ -458,12 +459,26 @@ class Meta: fields = [ "name", "notes", + "project_purl", ] widgets = { "name": forms.TextInput(attrs={"class": "input"}), "notes": forms.Textarea(attrs={"rows": 3, "class": "textarea is-dynamic"}), + "project_purl": forms.TextInput(attrs={"class": "input"}), } + def clean_project_purl(self): + """Validate the Project PURL.""" + project_purl = self.cleaned_data.get("project_purl") + + if project_purl: + try: + PackageURL.from_string(project_purl) + except ValueError: + raise forms.ValidationError("Project PURL must be a valid PackageURL") + + return project_purl + def __init__(self, *args, **kwargs): """Load initial values from Project ``settings`` field.""" super().__init__(*args, **kwargs) diff --git a/scanpipe/migrations/0068_project_project_purl.py b/scanpipe/migrations/0068_project_project_purl.py new file mode 100644 index 000000000..5d55137af --- /dev/null +++ b/scanpipe/migrations/0068_project_project_purl.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.7 on 2024-10-22 14:37 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('scanpipe', '0067_discoveredpackage_notes'), + ] + + operations = [ + migrations.AddField( + model_name='project', + name='project_purl', + field=models.CharField(blank=True, help_text='Project Package URL.', max_length=2048), + ), + ] diff --git a/scanpipe/models.py b/scanpipe/models.py index f54395b05..67abdf674 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -561,6 +561,11 @@ class Project(UUIDPKModel, ExtraDataFieldMixin, UpdateMixin, models.Model): notes = models.TextField(blank=True) settings = models.JSONField(default=dict, blank=True) labels = TaggableManager(through=UUIDTaggedItem) + project_purl = models.CharField( + max_length=2048, + blank=True, + help_text=_("Project Package URL."), + ) objects = ProjectQuerySet.as_manager() @@ -704,6 +709,7 @@ def clone( """Clone this project using the provided ``clone_name`` as new project name.""" new_project = Project.objects.create( name=clone_name, + project_purl=self.project_purl, settings=self.settings if copy_settings else {}, ) diff --git a/scanpipe/templates/scanpipe/project_settings.html b/scanpipe/templates/scanpipe/project_settings.html index d2ec52f58..b96ca8242 100644 --- a/scanpipe/templates/scanpipe/project_settings.html +++ b/scanpipe/templates/scanpipe/project_settings.html @@ -26,6 +26,14 @@ {{ form.name }} +
+ +
+ {{ form.project_purl }} +
+
- {{ form.project_purl }} + {{ form.purl }}
diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index f4513ac13..9c365fde9 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -2047,7 +2047,7 @@ def test_scanpipe_webhook_subscription_model_get_payload(self): "project": { "name": "Analysis", "uuid": str(self.project1.uuid), - "project_purl": "", + "purl": "", "is_archived": False, "notes": "", "labels": [], From b040aeb7b06fa474f12d9e3592ff6d8c6dee58dd Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 8 Nov 2024 19:52:49 +0530 Subject: [PATCH 13/16] Address reviews Signed-off-by: Keshav Priyadarshi --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5074f79c9..b0e183a33 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -22,7 +22,7 @@ v34.9.0 (unreleased) to FederatedCode. https://github.com/nexB/scancode.io/pull/1400 -- Add new ``project_purl`` field to project model. https://github.com/nexB/scancode.io/pull/1400 +- Add new ``purl`` field to project model. https://github.com/nexB/scancode.io/pull/1400 v34.8.3 (2024-10-30) -------------------- From cbcc5dce21d8470eb44e4784d6a726545b20c170 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 11 Nov 2024 22:00:59 +0530 Subject: [PATCH 14/16] Address review Signed-off-by: Keshav Priyadarshi --- scanpipe/forms.py | 2 +- scanpipe/templates/scanpipe/project_settings.html | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scanpipe/forms.py b/scanpipe/forms.py index 5ce265d72..c23300d19 100644 --- a/scanpipe/forms.py +++ b/scanpipe/forms.py @@ -486,7 +486,7 @@ class Meta: widgets = { "name": forms.TextInput(attrs={"class": "input"}), "notes": forms.Textarea(attrs={"rows": 3, "class": "textarea is-dynamic"}), - "purl": forms.TextInput(attrs={"class": "input"}), + "purl": forms.TextInput(attrs={"class": "input", "placeholder": "pkg:npm/lodash@4.0.1",}), } def clean_purl(self): diff --git a/scanpipe/templates/scanpipe/project_settings.html b/scanpipe/templates/scanpipe/project_settings.html index 8d6b556c6..731e9da0e 100644 --- a/scanpipe/templates/scanpipe/project_settings.html +++ b/scanpipe/templates/scanpipe/project_settings.html @@ -33,6 +33,7 @@
{{ form.purl }}
+

{{ form.purl.help_text }}