Introduce pnpm lockfile v9 detector (#1283)

* Introduce pnpm lockfile v9 detector

* Fix test build

* Add tests and update comments

* Update version

* coverage update

* requeue PR builds

* Fix smoke test

---------

Co-authored-by: Greg Villicana <[email protected]>

src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/PnpmYamlVersion.cs → src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/PnpmYaml.cs

@@ -2,7 +2,10 @@ namespace Microsoft.ComponentDetection.Detectors.Pnpm;
 
 using YamlDotNet.Serialization;
 
-public class PnpmYamlVersion
+/// <summary>
+/// Base class for all Pnpm lockfiles. Used for parsing the lockfile version.
+/// </summary>
+public class PnpmYaml
 {
     [YamlMember(Alias = "lockfileVersion")]
     public string LockfileVersion { get; set; }

src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/PnpmYamlV5.cs

@@ -6,14 +6,11 @@ namespace Microsoft.ComponentDetection.Detectors.Pnpm;
 /// <summary>
 /// Format for a Pnpm lock file version 5 as defined in https://github.com/pnpm/spec/blob/master/lockfile/5.md.
 /// </summary>
-public class PnpmYamlV5
+public class PnpmYamlV5 : PnpmYaml
 {
     [YamlMember(Alias = "dependencies")]
     public Dictionary<string, string> Dependencies { get; set; }
 
     [YamlMember(Alias = "packages")]
     public Dictionary<string, Package> Packages { get; set; }
-
-    [YamlMember(Alias = "lockfileVersion")]
-    public string LockfileVersion { get; set; }
 }

src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/PnpmHasDependenciesV6.cs → src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V6/PnpmHasDependenciesV6.cs

@@ -3,7 +3,7 @@ namespace Microsoft.ComponentDetection.Detectors.Pnpm;
 using System.Collections.Generic;
 using YamlDotNet.Serialization;
 
-public class PnpmHasDependenciesV6
+public class PnpmHasDependenciesV6 : PnpmYaml
 {
     [YamlMember(Alias = "dependencies")]
     public Dictionary<string, PnpmYamlV6Dependency> Dependencies { get; set; }

src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/PnpmYamlV6.cs → src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V6/PnpmYamlV6.cs

@@ -17,7 +17,4 @@ public class PnpmYamlV6 : PnpmHasDependenciesV6
 
     [YamlMember(Alias = "packages")]
     public Dictionary<string, Package> Packages { get; set; }
-
-    [YamlMember(Alias = "lockfileVersion")]
-    public string LockfileVersion { get; set; }
 }

src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V9/PnpmHasDependenciesV9.cs

@@ -0,0 +1,16 @@
+namespace Microsoft.ComponentDetection.Detectors.Pnpm;
+
+using System.Collections.Generic;
+using YamlDotNet.Serialization;
+
+public class PnpmHasDependenciesV9 : PnpmYaml
+{
+    [YamlMember(Alias = "dependencies")]
+    public Dictionary<string, PnpmYamlV9Dependency> Dependencies { get; set; }
+
+    [YamlMember(Alias = "devDependencies")]
+    public Dictionary<string, PnpmYamlV9Dependency> DevDependencies { get; set; }
+
+    [YamlMember(Alias = "optionalDependencies")]
+    public Dictionary<string, PnpmYamlV9Dependency> OptionalDependencies { get; set; }
+}

src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V9/PnpmYamlV9.cs

@@ -0,0 +1,21 @@
+namespace Microsoft.ComponentDetection.Detectors.Pnpm;
+
+using System.Collections.Generic;
+using YamlDotNet.Serialization;
+
+/// <summary>
+/// There is still no official docs for the new v9 lock if format, so these parsing contracts are empirically based.
+/// Issue tracking v9 specs: https://github.com/pnpm/spec/issues/6
+/// Format should eventually get updated here: https://github.com/pnpm/spec/blob/master/lockfile/6.0.md.
+/// </summary>
+public class PnpmYamlV9 : PnpmHasDependenciesV9
+{
+    [YamlMember(Alias = "importers")]
+    public Dictionary<string, PnpmHasDependenciesV9> Importers { get; set; }
+
+    [YamlMember(Alias = "packages")]
+    public Dictionary<string, Package> Packages { get; set; }
+
+    [YamlMember(Alias = "snapshots")]
+    public Dictionary<string, Package> Snapshots { get; set; }
+}

src/Microsoft.ComponentDetection.Detectors/pnpm/Contracts/V9/PnpmYamlV9Dependency.cs

@@ -0,0 +1,9 @@
+namespace Microsoft.ComponentDetection.Detectors.Pnpm;
+
+using YamlDotNet.Serialization;
+
+public class PnpmYamlV9Dependency
+{
+    [YamlMember(Alias = "version")]
+    public string Version { get; set; }
+}

src/Microsoft.ComponentDetection.Detectors/pnpm/IPnpmDetector.cs

@@ -14,3 +14,13 @@ public interface IPnpmDetector
     /// <param name="singleFileComponentRecorder">Component recorder to which to write the dependency graph.</param>
     public void RecordDependencyGraphFromFile(string yamlFileContent, ISingleFileComponentRecorder singleFileComponentRecorder);
 }
+
+/// <summary>
+/// Constants used in Pnpm Detectors.
+/// </summary>
+public static class PnpmConstants
+{
+    public const string PnpmFileDependencyPath = "file:";
+
+    public const string PnpmLinkDependencyPath = "link:";
+}

src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmParsingUtilitiesBase.cs

@@ -0,0 +1,52 @@
+namespace Microsoft.ComponentDetection.Detectors.Pnpm;
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using Microsoft.ComponentDetection.Contracts;
+using YamlDotNet.Serialization;
+
+public abstract class PnpmParsingUtilitiesBase<T>
+where T : PnpmYaml
+{
+    public T DeserializePnpmYamlFile(string fileContent)
+    {
+        var deserializer = new DeserializerBuilder()
+            .IgnoreUnmatchedProperties()
+            .Build();
+        return deserializer.Deserialize<T>(new StringReader(fileContent));
+    }
+
+    public virtual bool IsPnpmPackageDevDependency(Package pnpmPackage)
+    {
+        ArgumentNullException.ThrowIfNull(pnpmPackage);
+
+        return string.Equals(bool.TrueString, pnpmPackage.Dev, StringComparison.InvariantCultureIgnoreCase);
+    }
+
+    public bool IsLocalDependency(KeyValuePair<string, string> dependency)
+    {
+        // Local dependencies are dependencies that live in the file system
+        // this requires an extra parsing that is not supported yet
+        return dependency.Key.StartsWith(PnpmConstants.PnpmFileDependencyPath) || dependency.Value.StartsWith(PnpmConstants.PnpmFileDependencyPath) || dependency.Value.StartsWith(PnpmConstants.PnpmLinkDependencyPath);
+    }
+
+    /// <summary>
+    /// Parse a pnpm path of the form "/package-name/version".
+    /// </summary>
+    /// <param name="pnpmPackagePath">a pnpm path of the form "/package-name/version".</param>
+    /// <returns>Data parsed from path.</returns>
+    public abstract DetectedComponent CreateDetectedComponentFromPnpmPath(string pnpmPackagePath);
+
+    public virtual string ReconstructPnpmDependencyPath(string dependencyName, string dependencyVersion)
+    {
+        if (dependencyVersion.StartsWith('/'))
+        {
+            return dependencyVersion;
+        }
+        else
+        {
+            return $"/{dependencyName}@{dependencyVersion}";
+        }
+    }
+}

src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmParsingUtilitiesFactory.cs

@@ -0,0 +1,27 @@
+namespace Microsoft.ComponentDetection.Detectors.Pnpm;
+
+using System.IO;
+using YamlDotNet.Serialization;
+
+public static class PnpmParsingUtilitiesFactory
+{
+    public static PnpmParsingUtilitiesBase<T> Create<T>()
+    where T : PnpmYaml
+    {
+        return typeof(T).Name switch
+        {
+            nameof(PnpmYamlV5) => new PnpmV5ParsingUtilities<T>(),
+            nameof(PnpmYamlV6) => new PnpmV6ParsingUtilities<T>(),
+            nameof(PnpmYamlV9) => new PnpmV9ParsingUtilities<T>(),
+            _ => new PnpmV5ParsingUtilities<T>(),
+        };
+    }
+
+    public static string DeserializePnpmYamlFileVersion(string fileContent)
+    {
+        var deserializer = new DeserializerBuilder()
+            .IgnoreUnmatchedProperties()
+            .Build();
+        return deserializer.Deserialize<PnpmYaml>(new StringReader(fileContent))?.LockfileVersion;
+    }
+}

src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmV5ParsingUtilities.cs

@@ -0,0 +1,58 @@
+namespace Microsoft.ComponentDetection.Detectors.Pnpm;
+
+using System.Linq;
+using global::NuGet.Versioning;
+using Microsoft.ComponentDetection.Contracts;
+using Microsoft.ComponentDetection.Contracts.TypedComponent;
+
+public class PnpmV5ParsingUtilities<T> : PnpmParsingUtilitiesBase<T>
+where T : PnpmYaml
+{
+    public override DetectedComponent CreateDetectedComponentFromPnpmPath(string pnpmPackagePath)
+    {
+        var (parentName, parentVersion) = this.ExtractNameAndVersionFromPnpmPackagePath(pnpmPackagePath);
+        return new DetectedComponent(new NpmComponent(parentName, parentVersion));
+    }
+
+    private (string Name, string Version) ExtractNameAndVersionFromPnpmPackagePath(string pnpmPackagePath)
+    {
+        var pnpmComponentDefSections = pnpmPackagePath.Trim('/').Split('/');
+        (var packageVersion, var indexVersionIsAt) = this.GetPackageVersion(pnpmComponentDefSections);
+        if (indexVersionIsAt == -1)
+        {
+            // No version = not expected input
+            return (null, null);
+        }
+
+        var normalizedPackageName = string.Join("/", pnpmComponentDefSections.Take(indexVersionIsAt).ToArray());
+        return (normalizedPackageName, packageVersion);
+    }
+
+    private (string PackageVersion, int VersionIndex) GetPackageVersion(string[] pnpmComponentDefSections)
+    {
+        var indexVersionIsAt = -1;
+        var packageVersion = string.Empty;
+        var lastIndex = pnpmComponentDefSections.Length - 1;
+
+        // get version from packages with format /mute-stream/0.0.6
+        if (SemanticVersion.TryParse(pnpmComponentDefSections[lastIndex], out var _))
+        {
+            return (pnpmComponentDefSections[lastIndex], lastIndex);
+        }
+
+        // get version from packages with format /@babel/helper-compilation-targets/7.10.4_@[email protected]
+        var lastComponentSplit = pnpmComponentDefSections[lastIndex].Split("_");
+        if (SemanticVersion.TryParse(lastComponentSplit[0], out var _))
+        {
+            return (lastComponentSplit[0], lastIndex);
+        }
+
+        // get version from packages with format /sinon-chai/2.8.0/[email protected][email protected]
+        if (SemanticVersion.TryParse(pnpmComponentDefSections[lastIndex - 1], out var _))
+        {
+            return (pnpmComponentDefSections[lastIndex - 1], lastIndex - 1);
+        }
+
+        return (packageVersion, indexVersionIsAt);
+    }
+}

src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmV6ParsingUtilities.cs

@@ -0,0 +1,42 @@
+namespace Microsoft.ComponentDetection.Detectors.Pnpm;
+
+using System;
+using Microsoft.ComponentDetection.Contracts;
+using Microsoft.ComponentDetection.Contracts.TypedComponent;
+
+public class PnpmV6ParsingUtilities<T> : PnpmParsingUtilitiesBase<T>
+where T : PnpmYaml
+{
+    public override DetectedComponent CreateDetectedComponentFromPnpmPath(string pnpmPackagePath)
+    {
+        /*
+         * The format is documented at https://github.com/pnpm/spec/blob/master/dependency-path.md.
+         * At the writing it does not seem to reflect changes which were made in lock file format v6:
+         * See https://github.com/pnpm/spec/issues/5.
+         */
+
+        // Strip parenthesized suffices from package. These hold peed dep related information that is unneeded here.
+        // An example of a dependency path with these: /[email protected]([email protected])([email protected])([email protected])
+        var fullPackageNameAndVersion = pnpmPackagePath.Split("(")[0];
+
+        var packageNameParts = fullPackageNameAndVersion.Split("@");
+
+        // If package name contains `@` this will reconstruct it:
+        var fullPackageName = string.Join("@", packageNameParts[..^1]);
+
+        // Version is section after last `@`.
+        var packageVersion = packageNameParts[^1];
+
+        // Check for leading `/` from pnpm.
+        if (!fullPackageName.StartsWith('/'))
+        {
+            throw new FormatException("Found pnpm dependency path not starting with `/`. This case is currently unhandled.");
+        }
+
+        // Strip leading `/`.
+        // It is unclear if real packages could have a name starting with `/`, so avoid `TrimStart` that just in case.
+        var normalizedPackageName = fullPackageName[1..];
+
+        return new DetectedComponent(new NpmComponent(normalizedPackageName, packageVersion));
+    }
+}

src/Microsoft.ComponentDetection.Detectors/pnpm/ParsingUtilities/PnpmV9ParsingUtilities.cs

@@ -0,0 +1,54 @@
+namespace Microsoft.ComponentDetection.Detectors.Pnpm;
+
+using Microsoft.ComponentDetection.Contracts;
+using Microsoft.ComponentDetection.Contracts.TypedComponent;
+
+public class PnpmV9ParsingUtilities<T> : PnpmParsingUtilitiesBase<T>
+where T : PnpmYaml
+{
+    public override DetectedComponent CreateDetectedComponentFromPnpmPath(string pnpmPackagePath)
+    {
+        /*
+         * The format is documented at https://github.com/pnpm/spec/blob/master/dependency-path.md.
+         * At the writing it does not seem to reflect changes which were made in lock file format v9:
+         * See https://github.com/pnpm/spec/issues/5.
+         * In general, the spec sheet for the v9 lockfile is not published, so parsing of this lockfile was emperically determined.
+         * see https://github.com/pnpm/spec/issues/6
+         */
+
+        // Strip parenthesized suffices from package. These hold peed dep related information that is unneeded here.
+        // An example of a dependency path with these: /[email protected]([email protected])([email protected])([email protected])
+        var fullPackageNameAndVersion = pnpmPackagePath.Split("(")[0];
+
+        var packageNameParts = fullPackageNameAndVersion.Split("@");
+
+        // If package name contains `@` this will reconstruct it:
+        var fullPackageName = string.Join("@", packageNameParts[..^1]);
+
+        // Version is section after last `@`.
+        var packageVersion = packageNameParts[^1];
+
+        return new DetectedComponent(new NpmComponent(fullPackageName, packageVersion));
+    }
+
+    /// <summary>
+    /// Combine the information from a dependency edge in the dependency graph encoded in the ymal file into a full pnpm dependency path.
+    /// </summary>
+    /// <param name="dependencyName">The name of the dependency, as used as as the dictionary key in the yaml file when referring to the dependency.</param>
+    /// <param name="dependencyVersion">The final resolved version of the package for this dependency edge.
+    /// This includes details like which version of specific dependencies were specified as peer dependencies.
+    /// In some edge cases, such as aliased packages, this version may be an absolute dependency path, but the leading slash has been removed.
+    /// leaving the "dependencyName" unused, which is checked by whether there is an @ in the version. </param>
+    /// <returns>A pnpm dependency path for the specified version of the named package.</returns>
+    public override string ReconstructPnpmDependencyPath(string dependencyName, string dependencyVersion)
+    {
+        if (dependencyVersion.StartsWith('/') || dependencyVersion.Split("(")[0].Contains('@'))
+        {
+            return dependencyVersion;
+        }
+        else
+        {
+            return $"{dependencyName}@{dependencyVersion}";
+        }
+    }
+}