Merge pull request #49 from mr-russ/performance

Rebased performance parts of #23, passes unit tests

Author: Gianluca Arbezzano

PHPCtags.class.php

@@ -20,16 +20,18 @@ class PHPCtags
     );
 
     private $mParser;
-
-    private $mStructs;
-
+    private $mLines;
     private $mOptions;
+    private $tagdata;
+    private $cachefile;
+    private $filecount;
 
     public function __construct($options)
     {
         $this->mParser = new PHPParser_Parser(new PHPParser_Lexer);
-        $this->mStructs = array();
+        $this->mLines = array();
         $this->mOptions = $options;
+        $this->filecount = 0;
     }
 
     public function setMFile($file)
@@ -59,6 +61,10 @@ public function addFile($file)
         $this->mFiles[realpath($file)] = 1;
     }
 
+    public function setCacheFile($file) {
+        $this->cachefile = $file;
+    }
+
     public function addFiles($files)
     {
         foreach ($files as $file) {
@@ -250,10 +256,10 @@ private function struct($node, $reset=FALSE, $parent=array())
         return $structs;
     }
 
-    private function render()
+    private function render($structure)
     {
         $str = '';
-        foreach ($this->mStructs as $struct) {
+        foreach ($structure as $struct) {
             $file = $struct['file'];
 
             if (!in_array($struct['kind'], $this->mOptions['kinds'])) {
@@ -361,19 +367,42 @@ private function render()
             $str .= "\n";
         }
 
-        // remove the last line ending
-        $str = trim($str);
+        // remove the last line ending and carriage return
+        $str = trim(str_replace("\x0D", "", $str));
+
+        return $str;
+    }
+
+    private function full_render() {
+        // Files will have been rendered already, just join and export.
+
+        $str = '';
+        foreach($this->mLines as $file => $data) {
+          $str .= $data.PHP_EOL;
+        }
 
         // sort the result as instructed
         if (isset($this->mOptions['sort']) && ($this->mOptions['sort'] == 'yes' || $this->mOptions['sort'] == 'foldcase')) {
             $str = self::stringSortByLine($str, $this->mOptions['sort'] == 'foldcase');
         }
 
+        // Save all tag information to a file for faster updates if a cache file was specified.
+        if (isset($this->cachefile)) {
+            file_put_contents($this->cachefile, serialize($this->tagdata));
+            if ($this->mOptions['v']) {
+                echo "Saved cache file.".PHP_EOL;
+            }
+        }
+
+        $str = trim($str);
+
         return $str;
     }
 
     public function export()
     {
+        $start = microtime(true);
+
         if (empty($this->mFiles)) {
             throw new PHPCtagsException('No File specified.');
         }
@@ -382,11 +411,27 @@ public function export()
             $this->process($file);
         }
 
-        return $this->render();
+        $content = $this->full_render();
+
+        $end = microtime(true);
+
+        if ($this->mOptions['V']) {
+            echo "It tooks ".($end-$start)." seconds.".PHP_EOL;
+        }
+
+        return $content;
     }
 
     private function process($file)
     {
+        // Load the tag md5 data to skip unchanged files.
+        if (!isset($this->tagdata) && isset($this->cachefile) && file_exists(realpath($this->cachefile))) {
+            if ($this->mOptions['v']) {
+                echo "Loaded cache file.".PHP_EOL;
+            }
+            $this->tagdata = unserialize(file_get_contents(realpath($this->cachefile)));
+        }
+
         if (is_dir($file) && isset($this->mOptions['R'])) {
             $iterator = new RecursiveIteratorIterator(
                 new ReadableRecursiveDirectoryIterator(
@@ -408,32 +453,54 @@ private function process($file)
                 }
 
                 try {
-                    $this->setMFile((string) $filename);
-                    $this->mStructs = array_merge(
-                        $this->mStructs,
-                        $this->struct($this->mParser->parse(file_get_contents($this->mFile)), TRUE)
-                    );
+                    $this->process_single_file($filename);
                 } catch(Exception $e) {
                     echo "PHPParser: {$e->getMessage()} - {$filename}".PHP_EOL;
                 }
             }
         } else {
             try {
-                $this->setMFile($file);
-                $this->mStructs = array_merge(
-                    $this->mStructs,
-                    $this->struct($this->mParser->parse(file_get_contents($this->mFile)), TRUE)
-                );
+                $this->process_single_file($file);
             } catch(Exception $e) {
                 echo "PHPParser: {$e->getMessage()} - {$file}".PHP_EOL;
             }
         }
     }
+
+    private function process_single_file($filename)
+    {
+        if ($this->mOptions['v'] && $this->filecount > 1 && $this->filecount % 64 == 0) {
+            echo " ".$this->filecount." files".PHP_EOL;
+        }
+        $this->filecount++;
+
+        $this->setMFile((string) $filename);
+        $file = file_get_contents($this->mFile);
+        $md5 = md5($file);
+        if (isset($this->tagdata[$this->mFile][$md5])) {
+            // The file is the same as the previous time we analyzed and saved.
+            $this->mLines[$this->mFile] = $this->tagdata[$this->mFile][$md5];
+            if ($this->mOptions['V']) {
+                echo ".";
+            }
+            return;
+        }
+
+        $struct = $this->struct($this->mParser->parse($file), TRUE);
+        $this->mLines[$this->mFile] = $this->render($struct);
+        $this->tagdata[$this->mFile][$md5] = $this->mLines[$this->mFile];
+        if ($this->mOptions['debug']) {
+            echo "Parse: ".($finishfile - $startfile).", Merge: ".($finishmerge-$finishfile)."; (".$this->filecount.")".$this->mFile.PHP_EOL;
+        } else if ($this->mOptions['v']) {
+            echo "U";
+        }
+    }
+
 }
 
 class PHPCtagsException extends Exception {
     public function __toString() {
-        return "PHPCtags: {$this->message}\n";
+        return "\nPHPCtags: {$this->message}\n";
     }
 }
 

bootstrap.php

@@ -18,7 +18,7 @@
 Addresses: <techlivezheng@gmail.com>, https://github.com/techlivezheng/phpctags
 EOF;
 
-$options = getopt('af:Nno:RuV', array(
+$options = getopt('aC:f:Nno:RuV', array(
     'append::',
     'debug',
     'exclude:',
@@ -29,6 +29,7 @@
     'help',
     'recurse::',
     'sort::',
+    'verbose::',
     'version',
     'memory::',
 ));
@@ -42,12 +43,19 @@
   -f <name>
        Write tags to specified file. Value of "-" writes tags to stdout
        ["tags"].
+  -C <name>
+       Use a cache file to store tags for faster updates.
   -n   Equivalent to --excmd=number.
   -N   Equivalent to --excmd=pattern.
   -o   Alternative for -f.
   -R   Equivalent to --recurse.
   -u   Equivalent to --sort=no.
+<<<<<<< HEAD
+  -v   Equivalent to --verbose.
+  -V   Equivalent to --version.
+=======
   -V   Equivalent to --verbose.
+>>>>>>> a86869b... Fixed issues based on @mr-russ's PR.
   --append=[yes|no]
        Should tags should be appended to existing tag file [no]?
   --debug
@@ -72,12 +80,19 @@
        Recurse into directories supplied on command line [no].
   --sort=[yes|no|foldcase]
        Should tags be sorted (optionally ignoring case) [yes]?.
+<<<<<<< HEAD
+  --Version
+=======
+  --verbose=[yes|no]
+       Enable verbose messages describing actions on each source file.
   --version
+>>>>>>> a86869b... Fixed issues based on @mr-russ's PR.
        Print version identifier to standard output.
 EOF;
 
 // prune options and its value from the $argv array
 $argv_ = array();
+
 foreach ($options as $option => $value) {
   foreach ($argv as $key => $chunk) {
     $regex = '/^'. (isset($option[1]) ? '--' : '-') . $option . '/';
@@ -88,9 +103,17 @@
 }
 while ($key = array_pop($argv_)) unset($argv[$key]);
 
-// option -V is an alternative to --version
+// option -V is an alternative to --verbose
 if (isset($options['V'])) {
-    $options['version'] = FALSE;
+    $options['verbose'] = 'yes';
+}
+
+if (isset($options['verbose'])) {
+    if ($options['verbose'] === FALSE || yes_or_no($options['verbose']) == 'yes') {
+        $options['V'] = 'yes';
+    } else if (yes_or_no($options['verbose']) != 'no') {
+        die('phpctags: Invalid value for "verbose" option'.PHP_EOL);
+    }
 }
 
 if (!isset($options['debug'])) {
@@ -212,6 +235,7 @@
 try {
     $ctags = new PHPCtags($options);
     $ctags->addFiles($argv);
+    $ctags->setCacheFile(isset($options['C']) ? $options['C'] : null);
     $result = $ctags->export();
 } catch (Exception $e) {
     die("phpctags: {$e->getMessage()}".PHP_EOL);