Merge pull request #132 from microsoft/powershell-extract-implicit-models

PS: Extract source files found via `PSModulePath`

Author: MathiasVP

powershell/downgrades/c5191f89a6e3e8cea428b5c7326a06e335738533/old.dbscheme

@@ -0,0 +1,3 @@
+description: Add is_in_psmodule_path relation
+compatibility: full
+is_in_psmodule_path.rel: delete

powershell/downgrades/c5191f89a6e3e8cea428b5c7326a06e335738533/semmlecode.powershell.dbscheme

@@ -24,6 +24,9 @@ public override bool HandleFlag(string key, bool value)
                 case "dry-run":
                     SkipExtraction = value;
                     return true;
+                case "skip-psmodulepath-files":
+                    SkipPSModulePathFiles = value;
+                    return true;
                 default:
                     return base.HandleFlag(key, value);
             }
@@ -74,7 +77,7 @@ public override void InvalidArgument(string argument)
         /// <summary>
         /// List of extensions to include.
         /// </summary>
-        public IList<string> Extensions { get; } = new List<string>() { ".ps1" };
+        public IList<string> Extensions { get; } = new List<string>() { ".ps1", ".psd1" };
 
         /// <summary>
         /// Files/patterns to exclude.
@@ -127,6 +130,12 @@ private static FileInfo[] GetDefaultFiles()
         /// </summary>
         public bool SkipExtraction { get; private set; } = false;
 
+        /// <summary>
+        /// Whether to extract files in the paths found in the `PSModulePath`
+        /// environment variable.
+        /// </summary>
+        public bool SkipPSModulePathFiles { get; private set; } = false;
+
         /// <summary>
         /// Whether errors were encountered parsing the arguments.
         /// </summary>
@@ -158,13 +167,14 @@ public static void ShowHelp(System.IO.TextWriter output)
                 "PowerShell# standalone extractor\n\nExtracts PowerShell scripts in the current directory.\n"
             );
             output.WriteLine("Additional options:\n");
-            output.WriteLine("    <path>           Use the provided path instead.");
+            output.WriteLine("    <path>                       Use the provided path instead.");
             output.WriteLine(
-                "    --exclude:xxx    Exclude a file or directory (can be specified multiple times)"
+                "    --exclude:xxx                Exclude a file or directory (can be specified multiple times)"
             );
-            output.WriteLine("    --dry-run        Stop before extraction");
-            output.WriteLine("    --threads:nnn    Specify number of threads (default=CPU cores)");
-            output.WriteLine("    --verbose        Produce more output");
+            output.WriteLine("    --dry-run                    Stop before extraction");
+            output.WriteLine("    --threads:nnn                Specify number of threads (default=CPU cores)");
+            output.WriteLine("    --verbose                    Produce more output");
+            output.WriteLine("    --skip-psmodulepath-files    Avoid extracting source files in paths specified by the PSModulePath environment variable.");
         }
 
         private Options() { }

powershell/downgrades/c5191f89a6e3e8cea428b5c7326a06e335738533/upgrade.properties

@@ -45,7 +45,8 @@ public static int Main(string[] args)
 
             output.Log(Severity.Info, "Running PowerShell standalone extractor");
             var sourceFiles = options
-                .Files.Where(d =>
+                .Files.Concat(GetPSModuleFiles(options))
+                .Where(d =>
                     options.Extensions.Contains(
                         d.Extension,
                         StringComparer.InvariantCultureIgnoreCase
@@ -87,6 +88,30 @@ public static int Main(string[] args)
             return 0;
         }
 
+        private static String[] GetPSModulePaths()
+        {
+            return Environment.GetEnvironmentVariable("PSModulePath")?.Split(Path.PathSeparator)
+                ?? Array.Empty<string>();
+        }
+
+        private static IEnumerable<FileInfo> GetPSModuleFiles(Options options)
+        {
+            if(options.SkipPSModulePathFiles)
+            {
+                return Array.Empty<FileInfo>();
+            }
+            
+            return GetPSModulePaths()
+                .Where(d => Directory.Exists(d))
+                .SelectMany(d =>
+                {
+                    var di = new DirectoryInfo(d);
+                    return di.Exists
+                        ? di.GetFiles("*.*", SearchOption.AllDirectories)
+                        : new FileInfo[] { new(d) };
+                });
+        }
+
         private class ExtractionProgress : IProgressMonitor
         {
             public ExtractionProgress(ILogger output)

powershell/extractor/Semmle.Extraction.PowerShell.Standalone/Options.cs

@@ -2,6 +2,7 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 
 namespace Semmle.Extraction.PowerShell.Entities
 {
@@ -14,6 +15,18 @@ protected File(PowerShellContext cx, string path)
         {
         }
 
+        private static string[] GetPSModulePaths()
+        {
+            return Environment.GetEnvironmentVariable("PSModulePath")?.Split(Path.PathSeparator)
+                ?? Array.Empty<string>();
+        }
+
+        private bool PathIsInPSModulePath()
+        {
+            // Check if f's path is inside one of the paths in $Env:PSModulePath
+            return GetPSModulePaths().Any(originalPath.StartsWith);
+        }
+
         public override void Populate(TextWriter trapFile)
         {
             trapFile.files(this, TransformedPath.Value);
@@ -23,6 +36,11 @@ public override void Populate(TextWriter trapFile)
                 trapFile.containerparent(Extraction.Entities.Folder.Create(PowerShellContext, dir), this);
             }
 
+            if(PathIsInPSModulePath())
+            {
+                trapFile.is_in_psmodule_path(this);
+            }
+
             try
             {
                 System.Text.Encoding encoding;

powershell/extractor/Semmle.Extraction.PowerShell.Standalone/Program.cs

@@ -32,5 +32,9 @@ public static void locations_default(this System.IO.TextWriter trapFile, SourceL
         {
             trapFile.WriteTuple("locations_default", label, file, startLine, startCol, endLine, endCol);
         }
+
+        public static void is_in_psmodule_path(this System.IO.TextWriter trapFile, Entities.File file) {
+            trapFile.WriteTuple("is_in_psmodule_path", file);
+        }
     }
 }

powershell/extractor/Semmle.Extraction.PowerShell/Entities/Base/File.cs

@@ -209,6 +209,10 @@ class File extends Container, @file {
     not this.isStub()
   }
 
+  predicate isInPSModulePath() {
+    is_in_psmodule_path(this)
+  }
+
   /** Holds if this file is a library. */
   predicate fromLibrary() {
     not this.getBaseName() = "" and

powershell/extractor/Semmle.Extraction/Tuples.cs

@@ -41,6 +41,10 @@ containerparent(
   unique int child: @container ref
 );
 
+is_in_psmodule_path(
+  int file: @file ref
+);
+
 /* Comments */
 comment_entity(
   unique int id: @comment_entity,