Auto sampling. Directory exclusions

Author: morria

Dockerfile

@@ -1,20 +1,24 @@
 FROM php:8.2.2-cli
 
 RUN apt-get update --fix-missing
-RUN apt-get install -y git
 
 # nikic/php-ast
+RUN apt-get install -y git
 RUN cd /tmp; git clone https://github.com/nikic/php-ast.git && cd php-ast && phpize && ./configure && make && make install
 RUN echo "extension=ast" > /usr/local/etc/php/conf.d/docker-php-ast.ini
+RUN rm -rf /tmp/php-ast
+RUN apt-get purge -y git
 
 # Composer
 RUN cd /tmp; curl -O https://getcomposer.org/installer && php ./installer --install-dir=/usr/local/bin --filename=composer
+RUN rm /tmp/installer
 
 RUN mkdir -p /code2seq-paths-php
 WORKDIR /code2seq-paths-php
 
+# Copy the source and executable
 COPY vendor /code2seq-paths-php/vendor
 COPY src /code2seq-paths-php/src
 COPY bin /code2seq-paths-php/bin
 
-ENTRYPOINT ["/code2seq-paths-php/bin/code2seq-paths"]
+ENTRYPOINT ["/code2seq-paths-php/bin/code2seq-paths"]

src/Paths/FunctionPaths.php

@@ -15,28 +15,51 @@ class FunctionPaths
     private string $function_name;
 
     /** @var array<Path> */
-    private array $paths;
+    private array $paths = [];
+
+    private ?int $max_length = null;
+    private ?float $sample_rate = null;
 
     /** @param $paths array<Path> */
-    public function __construct(string $function_name, array $paths = [])
+    public function __construct(string $function_name, ?int $max_length = null)
     {
         $this->function_name = $function_name;
-        $this->paths = $paths;
+        $this->max_length = $max_length;
     }
 
     public function appendPath(Path $path): void
     {
-        $this->paths[] = $path;
+        if ($this->sample_rate === null) {
+            $this->paths[] = $path;
+        } else {
+            if (rand(0, 1000) / 1000.0 >= $this->sample_rate) {
+                $this->paths[] = $path;
+            }
+        }
+
+        // Once we break 2x the max paths, randomly cut it in half and start
+        // sampling at 50%
+        if ($this->sample_rate === null && $this->max_length !== null) {
+            if (count($this->paths) > (2 * $this->max_length)) {
+                // Set a sample rate of 50%
+                $this->sample_rate = 0.5;
+
+                // Trim off a random 50% of existing paths
+                $paths = $this->paths;
+                shuffle($paths);
+                $this->paths = array_values(array_slice($paths, 0, $this->max_length));
+            }
+        }
     }
 
     /**
      * @return \Generator<FunctionPaths>
      */
-    public static function fromFileName(string $file_name, bool $use_node_ids = false): \Generator
+    public static function fromFileName(string $file_name, bool $use_node_ids = false, ?int $max_length = null): \Generator
     {
         $ast = parse_file($file_name, get_version());
         foreach ((new FunctionScanner())($ast) as $function_ast) {
-            $function_path = new FunctionPaths($function_ast->children['name'] ?? 'anonymous');
+            $function_path = new FunctionPaths($function_ast->children['name'] ?? 'anonymous', $max_length);
             foreach ((new GraphNodeVisitor(null, $use_node_ids))($function_ast)->allTerminals() as $terminal) {
                 foreach ($terminal->allPathsToOtherTerminals() as $path) {
                     $function_path->appendPath($path);
@@ -51,16 +74,16 @@ public function isEmpty(): bool
         return empty($this->paths);
     }
 
-    public function toString(?int $max_length = null): string
+    public function toString(): string
     {
         if ($this->isEmpty()) {
             return '';
         }
 
         $paths = $this->paths;
-        if ($max_length !== null && count($paths) > $max_length) {
+        if ($this->max_length !== null && count($paths) > $this->max_length) {
             shuffle($paths);
-            $paths = array_slice($paths, 0, $max_length);
+            $paths = array_slice($paths, 0, $this->max_length);
         }
 
         return implode(' ', [

src/Paths/GraphNode/NonTerminal.php

@@ -58,16 +58,6 @@ public function allPathsToTerminals(PartialPath $prefix): \Generator
                 yield from $this->children[$i]->allPathsToTerminals($prefix);
             }
         }
-
-        /*
-        foreach ($this->children as $child) {
-            // Skip the node we just popped out of
-            if ($child === $previous_node) {
-                continue;
-            }
-            yield from $child->allPathsToTerminals($prefix);
-        }
-        */
     }
 
     /**

src/Paths/GraphNodeVisitor.php

@@ -21,7 +21,7 @@ public function __construct(GraphNode $parent = null, bool $use_node_ids = false
         $this->use_node_ids = $use_node_ids;
     }
 
-    public function newGraphNodeVisitorWithParent(GraphNode $parent = null)
+    public function newGraphNodeVisitorWithParent(GraphNode $parent = null): GraphNodeVisitor
     {
         return new GraphNodeVisitor($parent, $this->use_node_ids);
     }

src/Paths/Scan.php

@@ -1,4 +1,5 @@
 <?php
+
 declare(strict_types=1);
 
 namespace Paths;
@@ -8,9 +9,14 @@ final class Scan
     /**
      * 
      */
-    static function filesAndDirectories(array $files_and_directories, callable $callable): void
+    static function filesAndDirectories(array $files_and_directories, array $exclude_files_and_directories, callable $callable): void
     {
         foreach ($files_and_directories as $file_or_directory) {
+            foreach ($exclude_files_and_directories as $exclude_file_or_directory) {
+                if (strpos($file_or_directory, $exclude_file_or_directory) !== false) {
+                    continue 2;
+                }
+            }
             self::fileOrDirectory($file_or_directory, $callable);
         }
     }
@@ -48,4 +54,4 @@ static function file(string $file_name, callable $callable): void
     {
         $callable($file_name);
     }
-}
+}

src/paths.php

@@ -10,7 +10,7 @@
 use Paths\FunctionPaths;
 
 $rest_index = 0;
-$opts = getopt('h', ['help', 'max-length:', 'seed:', 'ids'], $rest_index);
+$opts = getopt('hl:s:ie:', ['help', 'max-length:', 'seed:', 'ids', 'exclude:'], $rest_index);
 
 if (!is_array($opts) || isset($opts['help'])) {
     echo <<<EOH
@@ -28,6 +28,9 @@
 
  -i, --ids
  Use IDs rather than names for context nodes. Defaults to false.
+
+ -e PATH, --exclude=PATH
+ Exclude the file or directory at the given path from being scanned.
  
  ...
  All other options will be treated as file names to
@@ -50,13 +53,22 @@
 
 $use_node_ids = isset($opts['ids']) || isset($opts['i']);
 
+$exclude_files_and_directories = array_merge(
+    isset($opts['exclude'])
+        ? (is_array($opts['exclude']) ? $opts['exclude'] : [$opts['exclude']])
+        : [],
+    isset($opts['e'])
+        ? (is_array($opts['e']) ? $opts['e'] : [$opts['e']])
+        : []
+);
+
 $files_and_directories = [];
 if ($rest_index > 0) {
     $rest = array_slice($argv, $rest_index);
     $files_and_directories += $rest;
 }
 
-Scan::filesAndDirectories($files_and_directories, function ($file_name) use ($max_length, $use_node_ids) {
+Scan::filesAndDirectories($files_and_directories, $exclude_files_and_directories, function ($file_name) use ($max_length, $use_node_ids) {
     if ('php' !== pathinfo($file_name, PATHINFO_EXTENSION)) {
         return;
     }