From c7be109504fe7f1b810f8fce24660d7684a0360c Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 27 Nov 2024 08:52:50 -0500 Subject: [PATCH 01/11] chore: change info format --- .../java/com/google/cloud/model/License.java | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/License.java b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/License.java index 246cff06b1..dfa81db415 100644 --- a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/License.java +++ b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/License.java @@ -40,11 +40,26 @@ public static License toLicense(String licenseStr) { try { return License.valueOf(value); } catch (IllegalArgumentException exception) { - LOGGER.log(Level.WARNING, String.format("%s is not recognized as any of the known license.", licenseStr)); + LOGGER.log(Level.WARNING, + String.format("%s is not recognized as any of the known license.", licenseStr)); return NOT_RECOGNIZED; } } + public boolean isCompliant() { + if (categories.isEmpty()) { + return false; + } + Set complaintLicenseCategories = LicenseCategory.compliantCategories(); + for (LicenseCategory category : categories) { + if (!complaintLicenseCategories.contains(category)) { + return false; + } + } + + return true; + } + public Set getCategories() { return ImmutableSet.copyOf(categories); } @@ -53,15 +68,9 @@ public Set getCategories() { public String toString() { String nonCompliantPrefix = "%s (Not Google-compliant!)"; String compliantPrefix = "%s (Google-compliant)"; - Set compliantCategories = LicenseCategory.compliantCategories(); - if (this.categories.isEmpty()) { - return String.format(nonCompliantPrefix, this.licenseStr); - } - for (LicenseCategory category : this.categories) { - if (!compliantCategories.contains(category)) { - return String.format(nonCompliantPrefix, this.licenseStr); - } - } - return String.format(compliantPrefix, this.licenseStr); + + return isCompliant() + ? String.format(compliantPrefix, this.licenseStr) + : String.format(nonCompliantPrefix, this.licenseStr); } } From 828bc569c56433b897e17a3e34d392e5aa3083b6 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 27 Nov 2024 09:21:02 -0500 Subject: [PATCH 02/11] only output non complaint license --- .../google/cloud/model/AnalysisResult.java | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/AnalysisResult.java b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/AnalysisResult.java index 006764055d..6c12fbc3bf 100644 --- a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/AnalysisResult.java +++ b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/AnalysisResult.java @@ -87,11 +87,20 @@ private String packageInfoReport() { } else { for (int i = 1; i < packageInfos.size(); i++) { PackageInfo info = packageInfos.get(i); - String dependencyInfo = String.format(""" - ### Package information of %s - %s - """, info.versionKey(), packageInfoSection(info)); - builder.append(dependencyInfo); + boolean hasNonComplaintLicenses = false; + for (License license : info.licenses()) { + if (!license.isCompliant()) { + hasNonComplaintLicenses = true; + break; + } + } + if (hasNonComplaintLicenses) { + builder.append(String.format("%s: %s", info.versionKey(), info.licenses())); + } + + for (Advisory advisory : info.advisories()) { + builder.append(String.format("%s: %s", info.versionKey(), advisory.url())); + } } } builder.append("\n"); From c52737486dddac61bd3e828a1ff9b876bf6fbf3b Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 27 Nov 2024 09:51:00 -0500 Subject: [PATCH 03/11] change workflow --- .github/workflows/analyze_dependency.yaml | 10 +-- .../dependency-analyzer/pom.xml | 5 ++ .../com/google/cloud/DependencyAnalyzer.java | 72 +++++++++++++------ .../java/com/google/cloud/model/License.java | 6 +- .../com/google/cloud/model/VersionKey.java | 4 ++ 5 files changed, 71 insertions(+), 26 deletions(-) diff --git a/.github/workflows/analyze_dependency.yaml b/.github/workflows/analyze_dependency.yaml index 22b350d53c..2b7d594b0f 100644 --- a/.github/workflows/analyze_dependency.yaml +++ b/.github/workflows/analyze_dependency.yaml @@ -27,10 +27,10 @@ jobs: distribution: temurin java-version: 17 cache: maven - - name: Set up Maven - uses: stCarolas/setup-maven@v4.5 - with: - maven-version: 3.8.2 + - name: Install modules + shell: bash + run: | + mvn clean install -V --batch-mode --no-transfer-progress -DskipTests - name: Install dependency analyzer shell: bash run: | @@ -39,5 +39,5 @@ jobs: - name: Check dependency information shell: bash run: | - mvn exec:java -Ddep.system=${{ github.event.inputs.system }} -Ddep.name=${{ github.event.inputs.name }} -Ddep.version=${{ github.event.inputs.version }} + mvn exec:java working-directory: java-shared-dependencies/dependency-analyzer \ No newline at end of file diff --git a/java-shared-dependencies/dependency-analyzer/pom.xml b/java-shared-dependencies/dependency-analyzer/pom.xml index fb2aa307ac..67cc0d8253 100644 --- a/java-shared-dependencies/dependency-analyzer/pom.xml +++ b/java-shared-dependencies/dependency-analyzer/pom.xml @@ -64,6 +64,11 @@ guava 33.3.1-jre + + com.google.cloud.tools + dependencies + 1.5.13 + org.mockito diff --git a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/DependencyAnalyzer.java b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/DependencyAnalyzer.java index 7423af8f11..369d224576 100644 --- a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/DependencyAnalyzer.java +++ b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/DependencyAnalyzer.java @@ -7,21 +7,28 @@ import com.google.cloud.model.AdvisoryKey; import com.google.cloud.model.AnalysisResult; import com.google.cloud.model.License; -import com.google.cloud.model.ReportResult; import com.google.cloud.model.PackageInfo; import com.google.cloud.model.QueryResult; +import com.google.cloud.model.ReportResult; import com.google.cloud.model.Result; import com.google.cloud.model.Version; import com.google.cloud.model.VersionKey; +import com.google.cloud.tools.opensource.classpath.ClassPathBuilder; +import com.google.cloud.tools.opensource.classpath.DependencyMediation; +import com.google.cloud.tools.opensource.dependencies.Bom; +import com.google.cloud.tools.opensource.dependencies.MavenRepositoryException; import java.io.IOException; import java.net.URISyntaxException; import java.net.http.HttpClient; +import java.nio.file.Paths; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Queue; import java.util.Set; +import org.eclipse.aether.artifact.Artifact; +import org.eclipse.aether.version.InvalidVersionSpecificationException; public class DependencyAnalyzer { @@ -31,9 +38,45 @@ public DependencyAnalyzer(DepsDevClient depsDevClient) { this.depsDevClient = depsDevClient; } - public AnalysisResult analyze(String system, String packageName, String packageVersion) - throws URISyntaxException, IOException, InterruptedException, IllegalArgumentException { - VersionKey root = VersionKey.from(system, packageName, packageVersion); + public AnalysisResult analyze(String bomPath) + throws URISyntaxException, IOException, InterruptedException { + List packageInfos = new ArrayList<>(); + try { + Set roots = getManagedDependenciesFromBom(Bom.readBom(Paths.get(bomPath))); + for (VersionKey versionKey : roots) { + if (versionKey.isSnapshot()) { + continue; + } + packageInfos.addAll(getPackageInfoFrom(versionKey)); + } + + } catch (MavenRepositoryException | InvalidVersionSpecificationException ex) { + System.out.printf("Caught exception when resolving dependencies from %s.", bomPath); + ex.printStackTrace(); + System.exit(1); + } + + return AnalysisResult.of(packageInfos); + } + + private static Set getManagedDependenciesFromBom(Bom bom) + throws InvalidVersionSpecificationException { + Set res = new HashSet<>(); + new ClassPathBuilder() + .resolve(bom.getManagedDependencies(), false, DependencyMediation.MAVEN) + .getClassPath() + .forEach( + classPath -> { + Artifact artifact = classPath.getArtifact(); + String pkg = String.format("%s:%s", artifact.getGroupId(), artifact.getArtifactId()); + res.add(VersionKey.from("MAVEN", pkg, artifact.getVersion())); + }); + + return res; + } + + private List getPackageInfoFrom(VersionKey root) + throws URISyntaxException, IOException, InterruptedException { Set seenPackage = new HashSet<>(); seenPackage.add(root); Queue queue = new ArrayDeque<>(); @@ -42,6 +85,9 @@ public AnalysisResult analyze(String system, String packageName, String packageV while (!queue.isEmpty()) { VersionKey versionKey = queue.poll(); dependencies.add(versionKey); + if (versionKey.toString().equals("org.graalvm.sdk:nativeimage:24.1.1")) { + continue; + } List directDependencies = depsDevClient.getDirectDependencies(versionKey); // only add unseen dependencies to the queue. directDependencies @@ -49,7 +95,6 @@ public AnalysisResult analyze(String system, String packageName, String packageV .filter(seenPackage::add) .forEach(queue::offer); } - List result = new ArrayList<>(); for (VersionKey versionKey : dependencies) { QueryResult packageInfo = depsDevClient.getQueryResult(versionKey); @@ -64,11 +109,10 @@ public AnalysisResult analyze(String system, String packageName, String packageV advisories.add(depsDevClient.getAdvisory(advisoryKey.id())); } } - result.add(new PackageInfo(versionKey, licenses, advisories)); } - return AnalysisResult.of(result); + return result; } /** @@ -88,23 +132,11 @@ public AnalysisResult analyze(String system, String packageName, String packageV * package management system. */ public static void main(String[] args) throws IllegalArgumentException { - checkArgument(args.length == 3, - """ - The length of the inputs should be 3. - The 1st input should be the package management system. - The 2nd input should be the package name. - The 3rd input should be the package version. - """ - ); - - String system = args[0]; - String packageName = args[1]; - String packageVersion = args[2]; DependencyAnalyzer dependencyAnalyzer = new DependencyAnalyzer( new DepsDevClient(HttpClient.newHttpClient())); AnalysisResult analyzeReport = null; try { - analyzeReport = dependencyAnalyzer.analyze(system, packageName, packageVersion); + analyzeReport = dependencyAnalyzer.analyze("java-shared-dependencies/pom.xml"); } catch (URISyntaxException | IOException | InterruptedException ex) { System.out.println( "Caught exception when fetching package information from https://deps.dev/"); diff --git a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/License.java b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/License.java index dfa81db415..9fbe24c7c2 100644 --- a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/License.java +++ b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/License.java @@ -1,6 +1,7 @@ package com.google.cloud.model; import static com.google.cloud.model.LicenseCategory.NOTICE; +import static com.google.cloud.model.LicenseCategory.PERMISSIVE; import static com.google.cloud.model.LicenseCategory.RESTRICTED; import com.google.common.collect.ImmutableSet; @@ -16,10 +17,13 @@ public enum License { APACHE_2_0("Apache-2.0", Set.of(NOTICE)), BCL("BCL", Set.of(RESTRICTED, NOTICE)), + BSD_2_CLAUSE("BSD-2-Clause", Set.of(NOTICE)), BSD_3_CLAUSE("BSD-3-Clause", Set.of(NOTICE)), GL2PS("GL2PS", Set.of(RESTRICTED, NOTICE)), + GPL_2_0_WITH_CLASSPATH_EXCEPTION("GPL-2.0-with-classpath-exception", Set.of(PERMISSIVE)), MIT("MIT", Set.of(NOTICE)), - NOT_RECOGNIZED("Not-Recognized", Set.of()); + NOT_RECOGNIZED("Not-Recognized", Set.of()), + UPL_1_0("UPL-1.0", Set.of(NOTICE)); private final static Logger LOGGER = Logger.getLogger(License.class.getName()); diff --git a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/VersionKey.java b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/VersionKey.java index 589ae7190a..ea59fd8bbe 100644 --- a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/VersionKey.java +++ b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/VersionKey.java @@ -20,6 +20,10 @@ public static VersionKey from(String system, String name, String version) return new VersionKey(pkg, name, version); } + public boolean isSnapshot() { + return version.endsWith("SNAPSHOT"); + } + @Override public String toString() { if (pkgManagement == PkgManagement.MAVEN) { From 710c87fc0360c3712c7652f428f6ac39a9434e02 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 27 Nov 2024 10:08:52 -0500 Subject: [PATCH 04/11] change format --- .../main/java/com/google/cloud/DependencyAnalyzer.java | 4 +--- .../main/java/com/google/cloud/model/AnalysisResult.java | 9 +++++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/DependencyAnalyzer.java b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/DependencyAnalyzer.java index 369d224576..3fdd97e01d 100644 --- a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/DependencyAnalyzer.java +++ b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/DependencyAnalyzer.java @@ -1,7 +1,5 @@ package com.google.cloud; -import static com.google.common.base.Preconditions.checkArgument; - import com.google.cloud.external.DepsDevClient; import com.google.cloud.model.Advisory; import com.google.cloud.model.AdvisoryKey; @@ -136,7 +134,7 @@ public static void main(String[] args) throws IllegalArgumentException { new DepsDevClient(HttpClient.newHttpClient())); AnalysisResult analyzeReport = null; try { - analyzeReport = dependencyAnalyzer.analyze("java-shared-dependencies/pom.xml"); + analyzeReport = dependencyAnalyzer.analyze("../pom.xml"); } catch (URISyntaxException | IOException | InterruptedException ex) { System.out.println( "Caught exception when fetching package information from https://deps.dev/"); diff --git a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/AnalysisResult.java b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/AnalysisResult.java index 6c12fbc3bf..66de117497 100644 --- a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/AnalysisResult.java +++ b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/AnalysisResult.java @@ -85,6 +85,7 @@ private String packageInfoReport() { if (packageInfos.size() == 1) { builder.append(String.format("%s has no dependency.", root.versionKey())); } else { + builder.append("==========Non-compliant licenses==========\n"); for (int i = 1; i < packageInfos.size(); i++) { PackageInfo info = packageInfos.get(i); boolean hasNonComplaintLicenses = false; @@ -95,11 +96,15 @@ private String packageInfoReport() { } } if (hasNonComplaintLicenses) { - builder.append(String.format("%s: %s", info.versionKey(), info.licenses())); + builder.append(String.format("%s: %s\n", info.versionKey(), info.licenses())); } + } + builder.append("==========Security vulnerabilities==========\n"); + for (int i = 1; i < packageInfos.size(); i++) { + PackageInfo info = packageInfos.get(i); for (Advisory advisory : info.advisories()) { - builder.append(String.format("%s: %s", info.versionKey(), advisory.url())); + builder.append(String.format("%s: %s\n", info.versionKey(), advisory.url())); } } } From 75f0cb4f0c75cc8f78f43ae68d90a3541f1eb80e Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 27 Nov 2024 10:42:37 -0500 Subject: [PATCH 05/11] add github pull request freshness --- .../com/google/cloud/DependencyAnalyzer.java | 24 ++++++++++++++++--- .../google/cloud/model/AnalysisResult.java | 2 ++ .../com/google/cloud/model/PackageInfo.java | 4 +++- .../com/google/cloud/model/ProjectKey.java | 23 ++++++++++++++++++ .../cloud/model/PullRequestStatistics.java | 10 +++++++- .../google/cloud/model/RelatedProject.java | 5 ++++ .../java/com/google/cloud/model/Version.java | 4 +++- 7 files changed, 66 insertions(+), 6 deletions(-) create mode 100644 java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/ProjectKey.java create mode 100644 java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/RelatedProject.java diff --git a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/DependencyAnalyzer.java b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/DependencyAnalyzer.java index 3fdd97e01d..22bab83b94 100644 --- a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/DependencyAnalyzer.java +++ b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/DependencyAnalyzer.java @@ -1,12 +1,16 @@ package com.google.cloud; import com.google.cloud.external.DepsDevClient; +import com.google.cloud.external.GitHubClient; import com.google.cloud.model.Advisory; import com.google.cloud.model.AdvisoryKey; import com.google.cloud.model.AnalysisResult; import com.google.cloud.model.License; import com.google.cloud.model.PackageInfo; +import com.google.cloud.model.ProjectKey; +import com.google.cloud.model.PullRequestStatistics; import com.google.cloud.model.QueryResult; +import com.google.cloud.model.RelatedProject; import com.google.cloud.model.ReportResult; import com.google.cloud.model.Result; import com.google.cloud.model.Version; @@ -23,6 +27,7 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.Queue; import java.util.Set; import org.eclipse.aether.artifact.Artifact; @@ -31,9 +36,11 @@ public class DependencyAnalyzer { private final DepsDevClient depsDevClient; + private final GitHubClient gitHubClient; - public DependencyAnalyzer(DepsDevClient depsDevClient) { + public DependencyAnalyzer(DepsDevClient depsDevClient, GitHubClient gitHubClient) { this.depsDevClient = depsDevClient; + this.gitHubClient = gitHubClient; } public AnalysisResult analyze(String bomPath) @@ -98,6 +105,7 @@ private List getPackageInfoFrom(VersionKey root) QueryResult packageInfo = depsDevClient.getQueryResult(versionKey); List licenses = new ArrayList<>(); List advisories = new ArrayList<>(); + Optional statistics = Optional.empty(); for (Result res : packageInfo.results()) { Version version = res.version(); for (String license : version.licenses()) { @@ -106,8 +114,17 @@ private List getPackageInfoFrom(VersionKey root) for (AdvisoryKey advisoryKey : version.advisoryKeys()) { advisories.add(depsDevClient.getAdvisory(advisoryKey.id())); } + + for (RelatedProject project : version.relatedProjects()) { + ProjectKey projectKey = project.projectKey(); + if (!projectKey.isGitHubProject()) { + continue; + } + statistics = Optional.of(gitHubClient.listMonthlyPullRequestStatusOf( + projectKey.organization(), projectKey.repo())); + } } - result.add(new PackageInfo(versionKey, licenses, advisories)); + result.add(new PackageInfo(versionKey, licenses, advisories, statistics)); } return result; @@ -131,7 +148,8 @@ private List getPackageInfoFrom(VersionKey root) */ public static void main(String[] args) throws IllegalArgumentException { DependencyAnalyzer dependencyAnalyzer = new DependencyAnalyzer( - new DepsDevClient(HttpClient.newHttpClient())); + new DepsDevClient(HttpClient.newHttpClient()), + new GitHubClient(HttpClient.newHttpClient())); AnalysisResult analyzeReport = null; try { analyzeReport = dependencyAnalyzer.analyze("../pom.xml"); diff --git a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/AnalysisResult.java b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/AnalysisResult.java index 66de117497..26eb48f11d 100644 --- a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/AnalysisResult.java +++ b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/AnalysisResult.java @@ -119,11 +119,13 @@ private String packageInfoSection(PackageInfo packageInfo) { String packageInfoReport = """ Licenses: %s Vulnerabilities: %s. + Pull request freshness: %s. Checked in [%s (%s)](%s) """; return String.format(packageInfoReport, packageInfo.licenses(), packageInfo.advisories(), + packageInfo.pullRequestStatistics().orElse(null), versionKey.name(), versionKey.version(), getQueryUrl( diff --git a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/PackageInfo.java b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/PackageInfo.java index 7aa1f12768..ad6a3f46fc 100644 --- a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/PackageInfo.java +++ b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/PackageInfo.java @@ -2,6 +2,7 @@ import com.google.common.collect.ImmutableList; import java.util.List; +import java.util.Optional; /** * Selected package information associated with a package version, including licenses and security @@ -10,7 +11,8 @@ public record PackageInfo( VersionKey versionKey, List licenses, - List advisories) { + List advisories, + Optional pullRequestStatistics) { public List licenses() { return ImmutableList.copyOf(licenses); diff --git a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/ProjectKey.java b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/ProjectKey.java new file mode 100644 index 0000000000..e05fc0a00e --- /dev/null +++ b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/ProjectKey.java @@ -0,0 +1,23 @@ +package com.google.cloud.model; + +/** + * The identifier for project. + * + * @param id A project identifier of the form github.com/user/repo, gitlab.com/user/repo, or + * bitbucket.org/user/repo. + */ +public record ProjectKey(String id) { + public boolean isGitHubProject() { + return id.startsWith("github.com"); + } + + public String organization() { + String[] strs = id.split("/"); + return strs[1]; + } + + public String repo() { + String[] strs = id.split("/"); + return strs[2]; + } +} diff --git a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/PullRequestStatistics.java b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/PullRequestStatistics.java index f151569f73..426d24c476 100644 --- a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/PullRequestStatistics.java +++ b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/PullRequestStatistics.java @@ -12,4 +12,12 @@ * @param merged The number of pull requests merged within the interval. * @param interval The time interval over which the statistics were collected. */ -public record PullRequestStatistics(long created, long merged, Interval interval) {} +public record PullRequestStatistics(long created, long merged, Interval interval) { + + @Override + public String toString() { + return String.format( + "%s pull requests are created and %s pull requests are merged in the last %s days", created, + merged, interval.getDays()); + } +} diff --git a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/RelatedProject.java b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/RelatedProject.java new file mode 100644 index 0000000000..04beccf605 --- /dev/null +++ b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/RelatedProject.java @@ -0,0 +1,5 @@ +package com.google.cloud.model; + +public record RelatedProject(ProjectKey projectKey) { + +} diff --git a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/Version.java b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/Version.java index 16acce8623..91ce020e22 100644 --- a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/Version.java +++ b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/Version.java @@ -13,11 +13,13 @@ * @param versionKey The name of the package version. * @param licenses The licenses governing the use of this package version. * @param advisoryKeys Security advisories known to affect this package version. + * @param relatedProjects Projects that are related to this package version. */ public record Version( VersionKey versionKey, List licenses, - List advisoryKeys) { + List advisoryKeys, + List relatedProjects) { public List advisoryKeys() { return ImmutableList.copyOf(advisoryKeys); From 950a07e2c87ddbaa1c9c5a1c62e3934c26a01f67 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 27 Nov 2024 10:50:14 -0500 Subject: [PATCH 06/11] fix test --- .../cloud/model/AnalysisResultTest.java | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/java-shared-dependencies/dependency-analyzer/src/test/java/com/google/cloud/model/AnalysisResultTest.java b/java-shared-dependencies/dependency-analyzer/src/test/java/com/google/cloud/model/AnalysisResultTest.java index ab3bd50b0e..d706dd5845 100644 --- a/java-shared-dependencies/dependency-analyzer/src/test/java/com/google/cloud/model/AnalysisResultTest.java +++ b/java-shared-dependencies/dependency-analyzer/src/test/java/com/google/cloud/model/AnalysisResultTest.java @@ -4,6 +4,7 @@ import static org.junit.Assert.assertEquals; import java.util.List; +import java.util.Optional; import org.junit.Test; public class AnalysisResultTest { @@ -23,7 +24,8 @@ public void testGenerateReportWithAdvisoriesReturnsFailure() new String[]{"CVE-2019-17571"}, 9.8, "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" - )) + )), + Optional.empty() ) ); ReportResult result = AnalysisResult.of(results).getAnalysisResult(); @@ -38,7 +40,8 @@ public void testGenerateReportWithNonCompliantLicenseReturnsFailure() new PackageInfo( root, List.of(License.toLicense("BCL")), - List.of() + List.of(), + Optional.empty() ) ); ReportResult result = AnalysisResult.of(results).getAnalysisResult(); @@ -53,7 +56,8 @@ public void testGenerateReportWithoutRiskSucceeds() new PackageInfo( root, List.of(License.toLicense("Apache-2.0")), - List.of() + List.of(), + Optional.empty() ) ); ReportResult result = AnalysisResult.of(results).getAnalysisResult(); @@ -67,17 +71,20 @@ public void testToStringReturnsNoRiskInformation() { new PackageInfo( root, List.of(License.toLicense("Apache-2.0")), - List.of() + List.of(), + Optional.empty() ), new PackageInfo( VersionKey.from("maven", "com.example:dependency", "4.5.6"), List.of(License.toLicense("Apache-2.0"), License.toLicense("MIT")), - List.of() + List.of(), + Optional.empty() ), new PackageInfo( VersionKey.from("maven", "com.example:nested-dependency", "2.3.1"), List.of(License.toLicense("Apache-2.0"), License.toLicense("MIT")), - List.of() + List.of(), + Optional.empty() ) ); assertEquals(""" @@ -108,7 +115,8 @@ public void testToStringReturnsRiskInformation() { new PackageInfo( root, List.of(License.APACHE_2_0, License.BCL), - List.of() + List.of(), + Optional.empty() ), new PackageInfo( VersionKey.from("maven", "com.example:dependency", "4.5.6"), @@ -120,12 +128,14 @@ public void testToStringReturnsRiskInformation() { new String[]{"CVE-2019-17571"}, 9.8, "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" - )) + )), + Optional.empty() ), new PackageInfo( VersionKey.from("maven", "com.example:nested-dependency", "2.3.1"), List.of(License.MIT, License.GL2PS), - List.of() + List.of(), + Optional.empty() ) ); assertEquals(""" From ecb40ed62109d3d5ad341ee376d52f33ae391988 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 27 Nov 2024 11:11:26 -0500 Subject: [PATCH 07/11] skip exception from github --- .../google/cloud/external/GitHubClient.java | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/external/GitHubClient.java b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/external/GitHubClient.java index 5c66a1af1a..4869f45d69 100644 --- a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/external/GitHubClient.java +++ b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/external/GitHubClient.java @@ -5,6 +5,7 @@ import com.google.cloud.model.PullRequestStatistics; import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import com.google.gson.JsonSyntaxException; import com.google.gson.reflect.TypeToken; import java.io.IOException; import java.net.URI; @@ -19,6 +20,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.logging.Logger; /** * GitHubClient is a class that sends HTTP requests to the GitHub RESTful API. It provides methods @@ -29,6 +31,8 @@ * requests and {@link com.google.gson.Gson} for handling JSON serialization/deserialization. */ public class GitHubClient { + + private final static Logger LOGGER = Logger.getLogger(GitHubClient.class.getName()); private final HttpClient client; private final Gson gson; private static final String PULL_REQUESTS_BASE = @@ -81,12 +85,20 @@ private List listPullRequests(String organization, String repo) List pullRequests = new ArrayList<>(); int page = 1; while (pullRequests.size() < MAX_PULL_REQUEST_NUM) { + System.out.println(getPullRequestsUrl(organization, repo, page)); HttpResponse response = getResponse(getPullRequestsUrl(organization, repo, page)); - pullRequests.addAll( - gson.fromJson(response.body(), new TypeToken>() {}.getType())); + try { + pullRequests.addAll( + gson.fromJson(response.body(), new TypeToken>() { + }.getType())); + } catch (JsonSyntaxException ex) { + LOGGER.warning(String.format( + "Can't parse response from GitHub API.\nOrganization: %s, repo: %s, page: %s\n", + organization, repo, page)); + } + page++; } - return pullRequests; } From 3500a9298e4df775063f80728d487ca8f188c605 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 27 Nov 2024 11:40:24 -0500 Subject: [PATCH 08/11] break --- .../src/main/java/com/google/cloud/external/GitHubClient.java | 1 + 1 file changed, 1 insertion(+) diff --git a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/external/GitHubClient.java b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/external/GitHubClient.java index 4869f45d69..1ebd777642 100644 --- a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/external/GitHubClient.java +++ b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/external/GitHubClient.java @@ -95,6 +95,7 @@ private List listPullRequests(String organization, String repo) LOGGER.warning(String.format( "Can't parse response from GitHub API.\nOrganization: %s, repo: %s, page: %s\n", organization, repo, page)); + break; } page++; From 7d08504d424daec45f86041676f80ce149d0d58f Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 27 Nov 2024 11:46:17 -0500 Subject: [PATCH 09/11] change output --- .../com/google/cloud/DependencyAnalyzer.java | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/DependencyAnalyzer.java b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/DependencyAnalyzer.java index 22bab83b94..27b4fe0ad5 100644 --- a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/DependencyAnalyzer.java +++ b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/DependencyAnalyzer.java @@ -43,16 +43,16 @@ public DependencyAnalyzer(DepsDevClient depsDevClient, GitHubClient gitHubClient this.gitHubClient = gitHubClient; } - public AnalysisResult analyze(String bomPath) + public List analyze(String bomPath) throws URISyntaxException, IOException, InterruptedException { - List packageInfos = new ArrayList<>(); + List analysisResults = new ArrayList<>(); try { Set roots = getManagedDependenciesFromBom(Bom.readBom(Paths.get(bomPath))); for (VersionKey versionKey : roots) { if (versionKey.isSnapshot()) { continue; } - packageInfos.addAll(getPackageInfoFrom(versionKey)); + analysisResults.add(AnalysisResult.of(getPackageInfoFrom(versionKey))); } } catch (MavenRepositoryException | InvalidVersionSpecificationException ex) { @@ -61,7 +61,7 @@ public AnalysisResult analyze(String bomPath) System.exit(1); } - return AnalysisResult.of(packageInfos); + return analysisResults; } private static Set getManagedDependenciesFromBom(Bom bom) @@ -150,9 +150,9 @@ public static void main(String[] args) throws IllegalArgumentException { DependencyAnalyzer dependencyAnalyzer = new DependencyAnalyzer( new DepsDevClient(HttpClient.newHttpClient()), new GitHubClient(HttpClient.newHttpClient())); - AnalysisResult analyzeReport = null; + List analysisResults = null; try { - analyzeReport = dependencyAnalyzer.analyze("../pom.xml"); + analysisResults = dependencyAnalyzer.analyze("../pom.xml"); } catch (URISyntaxException | IOException | InterruptedException ex) { System.out.println( "Caught exception when fetching package information from https://deps.dev/"); @@ -161,13 +161,9 @@ public static void main(String[] args) throws IllegalArgumentException { } System.out.println("Please copy and paste the package information below to your ticket.\n"); - System.out.println(analyzeReport.toString()); - ReportResult result = analyzeReport.getAnalysisResult(); - System.out.println(result); - if (result.equals(ReportResult.FAIL)) { - System.out.println( - "Please refer to go/cloud-java-rotations#security-advisories-monitoring for further actions"); - System.exit(1); - } + analysisResults.forEach(analysisResult -> { + System.out.println(analysisResult.toString()); + System.out.println(analysisResult.getAnalysisResult()); + }); } } From a5f4cd9ccf1c3738871290ff5b7d3ed6a2a5e927 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 27 Nov 2024 13:05:09 -0500 Subject: [PATCH 10/11] write to csv --- .../dependency-analyzer/pom.xml | 5 +++ .../com/google/cloud/DependencyAnalyzer.java | 35 +++++++++++++------ .../google/cloud/model/AnalysisResult.java | 13 +++++++ 3 files changed, 43 insertions(+), 10 deletions(-) diff --git a/java-shared-dependencies/dependency-analyzer/pom.xml b/java-shared-dependencies/dependency-analyzer/pom.xml index 67cc0d8253..c47053de2f 100644 --- a/java-shared-dependencies/dependency-analyzer/pom.xml +++ b/java-shared-dependencies/dependency-analyzer/pom.xml @@ -69,6 +69,11 @@ dependencies 1.5.13 + + com.opencsv + opencsv + 5.9 + org.mockito diff --git a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/DependencyAnalyzer.java b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/DependencyAnalyzer.java index 27b4fe0ad5..9701695605 100644 --- a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/DependencyAnalyzer.java +++ b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/DependencyAnalyzer.java @@ -19,6 +19,9 @@ import com.google.cloud.tools.opensource.classpath.DependencyMediation; import com.google.cloud.tools.opensource.dependencies.Bom; import com.google.cloud.tools.opensource.dependencies.MavenRepositoryException; +import com.opencsv.CSVWriter; +import java.io.File; +import java.io.FileWriter; import java.io.IOException; import java.net.URISyntaxException; import java.net.http.HttpClient; @@ -115,14 +118,14 @@ private List getPackageInfoFrom(VersionKey root) advisories.add(depsDevClient.getAdvisory(advisoryKey.id())); } - for (RelatedProject project : version.relatedProjects()) { - ProjectKey projectKey = project.projectKey(); - if (!projectKey.isGitHubProject()) { - continue; - } - statistics = Optional.of(gitHubClient.listMonthlyPullRequestStatusOf( - projectKey.organization(), projectKey.repo())); - } + // for (RelatedProject project : version.relatedProjects()) { + // ProjectKey projectKey = project.projectKey(); + // if (!projectKey.isGitHubProject()) { + // continue; + // } + // statistics = Optional.of(gitHubClient.listMonthlyPullRequestStatusOf( + // projectKey.organization(), projectKey.repo())); + // } } result.add(new PackageInfo(versionKey, licenses, advisories, statistics)); } @@ -146,7 +149,7 @@ private List getPackageInfoFrom(VersionKey root) * @throws IllegalArgumentException if the format of package name is incorrect according to the * package management system. */ - public static void main(String[] args) throws IllegalArgumentException { + public static void main(String[] args) throws IllegalArgumentException, IOException { DependencyAnalyzer dependencyAnalyzer = new DependencyAnalyzer( new DepsDevClient(HttpClient.newHttpClient()), new GitHubClient(HttpClient.newHttpClient())); @@ -161,9 +164,21 @@ public static void main(String[] args) throws IllegalArgumentException { } System.out.println("Please copy and paste the package information below to your ticket.\n"); + // create CSVWriter object filewriter object as parameter + CSVWriter writer = new CSVWriter(new FileWriter("dependency-report.csv")); + writer.writeNext(new String[]{"Dependency", "License problem", "Vulnerabilities"}); analysisResults.forEach(analysisResult -> { System.out.println(analysisResult.toString()); - System.out.println(analysisResult.getAnalysisResult()); + String licenseProblem = ""; + if (!analysisResult.getNonCompliantLicenses().isEmpty()) { + licenseProblem = analysisResult.getNonCompliantLicenses().toString(); + } + String advisories = ""; + if (!analysisResult.getAdvisories().isEmpty()) { + advisories = analysisResult.getAdvisories().toString(); + } + writer.writeNext( + new String[]{analysisResult.getRoot().toString(), licenseProblem, advisories}); }); } } diff --git a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/AnalysisResult.java b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/AnalysisResult.java index 26eb48f11d..7f96fb11c0 100644 --- a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/AnalysisResult.java +++ b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/model/AnalysisResult.java @@ -3,6 +3,7 @@ import static com.google.cloud.external.DepsDevClient.QUERY_URL_BASE; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -20,6 +21,18 @@ private AnalysisResult(List result) { this.nonCompliantLicenses = getNonCompliantLicenses(result); } + public Map> getNonCompliantLicenses() { + return Collections.unmodifiableMap(nonCompliantLicenses); + } + + public Map> getAdvisories() { + return Collections.unmodifiableMap(advisories); + } + + public VersionKey getRoot() { + return packageInfos.get(0).versionKey(); + } + public static AnalysisResult of(List result) { return new AnalysisResult(result); } From bb3a1121f1c48c528067046f49558af9359540e9 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 27 Nov 2024 13:08:26 -0500 Subject: [PATCH 11/11] write to csv --- .../src/main/java/com/google/cloud/DependencyAnalyzer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/DependencyAnalyzer.java b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/DependencyAnalyzer.java index 9701695605..0cc7fb535f 100644 --- a/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/DependencyAnalyzer.java +++ b/java-shared-dependencies/dependency-analyzer/src/main/java/com/google/cloud/DependencyAnalyzer.java @@ -180,5 +180,6 @@ public static void main(String[] args) throws IllegalArgumentException, IOExcept writer.writeNext( new String[]{analysisResult.getRoot().toString(), licenseProblem, advisories}); }); + writer.close(); } }