Improve Performance, reduce memory footprint

array_merge has performance issues as the input array's grow in size.
mStructs is a complex array and can be a large memory user in php.

mStructs was replaced with mLines which is the processed tag information
to reduce the memory footprint and allow caching of the results.

mLines is an array of files which can be joined together or appended
to an existing file.

Performance seen from changes;
 - Moodle 2.4 codebase can be completely scanned in 3 minutes (approximate 1 hour before)

Author: Russell Smith

PHPCtags.class.php

@@ -20,16 +20,17 @@ class PHPCtags
     );
 
     private $mParser;
-
-    private $mStructs;
-
+    private $mLines;
     private $mOptions;
+    private $tagdata;
+    private $filecount;
 
     public function __construct($options)
     {
         $this->mParser = new PHPParser_Parser(new PHPParser_Lexer);
-        $this->mStructs = array();
+        $this->mLines = array();
         $this->mOptions = $options;
+        $this->filecount = 0;
     }
 
     public function setMFile($file)
@@ -250,10 +251,10 @@ private function struct($node, $reset=FALSE, $parent=array())
         return $structs;
     }
 
-    private function render()
+    private function render($structure)
     {
         $str = '';
-        foreach ($this->mStructs as $struct) {
+        foreach ($structure as $struct) {
             $file = $struct['file'];
 
             if (!isset($files[$file]))
@@ -360,6 +361,17 @@ private function render()
         // remove the last line ending
         $str = trim($str);
 
+        return $str;
+    }
+
+    private function full_render() {
+        // Files will have been rendered already, just join and export.
+
+        $str = '';
+        foreach($this->mLines as $file => $data) {
+          $str .= $data;
+        }
+
         // sort the result as instructed
         if (isset($this->mOptions['sort']) && ($this->mOptions['sort'] == 'yes' || $this->mOptions['sort'] == 'foldcase')) {
             $str = self::stringSortByLine($str, $this->mOptions['sort'] == 'foldcase');
@@ -378,7 +390,7 @@ public function export()
             $this->process($file);
         }
 
-        return $this->render();
+        return $this->full_render();
     }
 
     private function process($file)
@@ -404,27 +416,53 @@ private function process($file)
                 }
 
                 try {
-                    $this->setMFile((string) $filename);
-                    $this->mStructs = array_merge(
-                        $this->mStructs,
-                        $this->struct($this->mParser->parse(file_get_contents($this->mFile)), TRUE)
-                    );
+                    $this->process_single_file($filename);
                 } catch(Exception $e) {
                     echo "PHPParser: {$e->getMessage()} - {$filename}".PHP_EOL;
                 }
             }
         } else {
             try {
-                $this->setMFile($file);
-                $this->mStructs = array_merge(
-                    $this->mStructs,
-                    $this->struct($this->mParser->parse(file_get_contents($this->mFile)), TRUE)
-                );
+                    $this->process_single_file($filename);
             } catch(Exception $e) {
                 echo "PHPParser: {$e->getMessage()} - {$file}".PHP_EOL;
             }
         }
     }
+
+    private function process_single_file($filename)
+    {
+        if ($this->mOptions['v'] && $this->filecount > 1 && $this->filecount % 64 == 0) {
+            echo " ".$this->filecount." files".PHP_EOL;
+        }
+        $this->filecount++;
+
+        $startfile = microtime(true);
+
+        $this->setMFile((string) $filename);
+        $file = file_get_contents($this->mFile);
+        $md5 = md5($file);
+        if (isset($this->tagdata[$this->mFile][$md5])) {
+            // The file is the same as the previous time we analyzed and saved.
+            $this->mLines[$this->mFile] = $this->tagdata[$this->mFile][$md5];
+            if ($this->mOptions['v']) {
+                echo ".";
+            }
+            return;
+        }
+
+        $struct = $this->struct($this->mParser->parse($file), TRUE);
+        $finishfile = microtime(true);
+        $this->mLines[$this->mFile] = $this->render($struct);
+        $finishmerge = microtime(true);
+        $this->tagdata[$this->mFile][$md5] = $this->mLines[$this->mFile];
+        if ($this->mOptions['debug']) {
+            echo "Parse: ".($finishfile - $startfile).", Merge: ".($finishmerge-$finishfile)."; (".$this->filecount.")".$this->mFile.PHP_EOL;
+        } else if ($this->mOptions['v']) {
+            echo "U";
+        }
+    }
+
 }
 
 class PHPCtagsException extends Exception {

bootstrap.php

@@ -18,7 +18,7 @@
 Addresses: <techlivezheng@gmail.com>, https://github.com/techlivezheng/phpctags
 EOF;
 
-$options = getopt('af:Nno:RuV', array(
+$options = getopt('aC:f:Nno:RuvV', array(
     'append::',
     'debug',
     'exclude:',
@@ -46,7 +46,8 @@
   -o   Alternative for -f.
   -R   Equivalent to --recurse.
   -u   Equivalent to --sort=no.
-  -V   Equivalent to --verbose.
+  -v   Equivalent to --verbose.
+  -V   Equivalent to --version.
   --append=[yes|no]
        Should tags should be appended to existing tag file [no]?
   --debug
@@ -69,12 +70,13 @@
        Recurse into directories supplied on command line [no].
   --sort=[yes|no|foldcase]
        Should tags be sorted (optionally ignoring case) [yes]?.
-  --version
+  --Version
        Print version identifier to standard output.
 EOF;
 
 // prune options and its value from the $argv array
 $argv_ = array();
+
 foreach ($options as $option => $value) {
   foreach ($argv as $key => $chunk) {
     $regex = '/^'. (isset($option[1]) ? '--' : '-') . $option . '/';