Merge pull request #24 from Automattic/add/row-iterator

Add a row iterator

Author: xyu

README.md

@@ -11,27 +11,95 @@ Currently the following engines are supported:
 Usage Example
 -------------
 
+The recommended way to use this library is to get results from Hive/Impala via the memory efficient iterator which will keep the connection open and scroll through the results a couple rows at a time. This allows the processing of large result datasets one record at a time minimizing PHP's memory consumption.
+
 ```php
 // Load this lib
 require_once __DIR__ . '/ThriftSQL.phar';
 
-// Try out a Hive query
+// Try out a Hive query via iterator object
+$hive = new \ThriftSQL\Hive( 'hive.host.local', 10000, 'user', 'pass' );
+$hiveTables = $hive
+  ->connect()
+  ->getIterator( 'SHOW TABLES' );
+
+// Try out an Impala query via iterator object
+$impala = new \ThriftSQL\Impala( 'impala.host.local' );
+$impalaTables = $impala
+  ->connect()
+  ->getIterator( 'SHOW TABLES' );
+
+// Execute the Hive query and iterate over the result set
+foreach( $hiveTables as $rowNum => $row ) {
+  print_r( $row );
+}
+
+// Execute the Impala query and iterate over the result set
+foreach( $impalaTables as $rowNum => $row ) {
+  print_r( $row );
+}
+
+// Don't forget to close socket connection once you're done with it
+$hive->disconnect();
+$impala->disconnect();
+```
+
+The downside to using the memory efficient iterator is that we can only iterate over the result set once. If a second `foreach` is called on the same iterator object an exception is thrown by default to prevent the same query from executing on Hive/Impala again as results are not cached within the PHP client. This can be turned off however be aware iterating over the same iterator object may produce different results as the query is rerun.
+
+Consider the following example:
+
+```php
+// Connect to hive and get a rerun-able iterator
+$hive = new \ThriftSQL\Hive( 'hive.host.local', 10000, 'user', 'pass' );
+$results = $hive
+  ->connect()
+  ->getIterator( 'SELECT UNIX_TIMESTAMP()' )
+  ->allowRerun( true );
+
+// Execute the Hive query and get results
+foreach( $results as $rowNum => $row ) {
+  echo "Hive server time is: {$v[0]}\n";
+}
+
+sleep(3);
+
+// Execute the Hive query a second time
+foreach( $results as $rowNum => $row ) {
+  echo "Hive server time is: {$v[0]}\n";
+}
+```
+
+Which will output something like:
+
+```
+Hive server time is: 1517875200
+Hive server time is: 1517875203
+```
+
+If the result set is small and it would be easier to load all of it into PHP memory the `queryAndFetchAll()` method can be used which will return a plain numeric multidimensional array of the full result set.
+
+```php
+// Try out a small Hive query
 $hive = new \ThriftSQL\Hive( 'hive.host.local', 10000, 'user', 'pass' );
 $hiveTables = $hive
   ->connect()
   ->queryAndFetchAll( 'SHOW TABLES' );
+$hive->disconnect();
+
+// Print out the cached results
 print_r( $hiveTables );
+```
 
-// Try out an Impala query
+```php
+// Try out a small Impala query
 $impala = new \ThriftSQL\Impala( 'impala.host.local' );
 $impalaTables = $impala
   ->connect()
   ->queryAndFetchAll( 'SHOW TABLES' );
-print_r( $impalaTables );
-
-// Don't forget to clear the client and close socket.
-$hive->disconnect();
 $impala->disconnect();
+
+// Print out the cached results
+print_r( $impalaTables );
 ```
 
 Developing & Contributing

ThriftSQL.phar

@@ -1,11 +1,11 @@
 <?php
 
-interface ThriftSQL {
+abstract class ThriftSQL {
   /**
   * @return self
   * @throws \ThriftSQL\Exception
   */
-  public function connect();
+  abstract public function connect();
 
   /**
   * The simplest use case; takes a query string executes it synchronously and
@@ -15,7 +15,7 @@ public function connect();
   * @return array
   * @throws \ThriftSQL\Exception
   */
-  public function queryAndFetchAll( $queryStr );
+  abstract public function queryAndFetchAll( $queryStr );
 
   /**
   * Sends a query string for execution on the server and returns a
@@ -25,10 +25,21 @@ public function queryAndFetchAll( $queryStr );
   * @return \ThriftSQLQuery
   * @throws \ThriftSQL\Exception
   */
-  public function query( $queryStr );
+  abstract public function query( $queryStr );
 
   /**
   * @return null
   */
-  public function disconnect();
+  abstract public function disconnect();
+
+  /**
+   * Gets a memory efficient iterator that you can use in a foreach loop.
+   *
+   * @param string $queryStr
+   * @return \ThriftSQL\Utils\Iterator
+   * @throws \ThriftSQL\Exception
+   */
+  public function getIterator( $queryStr ) {
+    return new \ThriftSQL\Utils\Iterator( $this, $queryStr );
+  }
 }

src/ThriftSQL.php

@@ -2,7 +2,7 @@
 
 namespace ThriftSQL;
 
-class Hive implements \ThriftSQL {
+class Hive extends \ThriftSQL {
   private $_host;
   private $_port;
   private $_username;
@@ -12,7 +12,7 @@ class Hive implements \ThriftSQL {
   private $_client;
   private $_sessionHandle;
   protected $_sasl = true;
-  
+
   public function __construct( $host, $port = 10000, $username = null, $password = null, $timeout = null ) {
     $this->_host = $host;
     $this->_port = $port;

src/ThriftSQL/Hive.php

@@ -121,4 +121,4 @@ private function _isOperationRunning( $state ) {
       )
     );
   }
-}
+}

src/ThriftSQL/HiveQuery.php

@@ -1,7 +1,7 @@
 <?php
 namespace ThriftSQL;
 
-class Impala implements \ThriftSQL {
+class Impala extends \ThriftSQL {
 
   private $_host;
   private $_port;

src/ThriftSQL/Impala.php

@@ -20,7 +20,7 @@ public function __construct( $queryStr, $client ) {
 
   public function wait() {
     $sleeper = new \ThriftSQL\Utils\Sleeper();
-    
+
     // Wait for results
     $sleeper->reset();
     do {
@@ -134,4 +134,4 @@ private function _isOperationRunning( $state ) {
       )
     );
   }
-}
+}

src/ThriftSQL/ImpalaQuery.php

@@ -0,0 +1,104 @@
+<?php
+
+namespace ThriftSQL\Utils;
+
+class Iterator implements \Iterator {
+
+  const BUFFER_ROWS = 64;
+
+  /**
+   * @var \ThriftSQL
+   */
+  private $thriftSQL;
+
+  /**
+   * @var \ThriftSQLQuery
+   */
+  private $thriftSQLQuery;
+
+  private $queryStr;
+  private $buffer;
+  private $location;
+
+  private $runCount = 0;
+  private $allowRerun = false;
+
+  public function __construct( \ThriftSQL $thriftSQL, $queryStr ) {
+    $this->thriftSQL = $thriftSQL;
+    $this->queryStr = $queryStr;
+  }
+
+  public function allowRerun( $value ) {
+    $this->allowRerun = (bool) $value;
+    return $this;
+  }
+
+  /**
+   * Return the current element
+   * @link http://php.net/manual/en/iterator.current.php
+   * @return mixed Can return any type.
+   * @since 5.0.0
+   */
+  public function current() {
+    return $this->buffer[0];
+  }
+
+  /**
+   * Move forward to next element
+   * @link http://php.net/manual/en/iterator.next.php
+   * @return void Any returned value is ignored.
+   * @since 5.0.0
+   */
+  public function next() {
+    $this->location++;
+    array_shift( $this->buffer );
+  }
+
+  /**
+   * Return the key of the current element
+   * @link http://php.net/manual/en/iterator.key.php
+   * @return mixed scalar on success, or null on failure.
+   * @since 5.0.0
+   */
+  public function key() {
+    return $this->location;
+  }
+
+  /**
+   * Checks if current position is valid
+   * @link http://php.net/manual/en/iterator.valid.php
+   * @return boolean The return value will be casted to boolean and then evaluated.
+   * Returns true on success or false on failure.
+   * @since 5.0.0
+   */
+  public function valid() {
+    if ( ! empty( $this->buffer ) ) {
+      return true;
+    }
+
+    $this->buffer = $this->thriftSQLQuery->fetch( self::BUFFER_ROWS );
+    return ( ! empty( $this->buffer ) );
+  }
+
+  /**
+   * Rewind the Iterator to the first element
+   * @link http://php.net/manual/en/iterator.rewind.php
+   * @return void Any returned value is ignored.
+   * @since 5.0.0
+   * @throws \ThriftSQL\Exception
+   */
+  public function rewind() {
+    if ( $this->runCount > 0 && !$this->allowRerun ) {
+      throw new \ThriftSQL\Exception(
+        'Iterator rewound, this will cause the ThriftSQL to execute again. ' .
+        'Set `' . __CLASS__ . '::allowRerun(true)` to allow this behavior.'
+      );
+    }
+    $this->runCount++;
+
+    $this->thriftSQLQuery = $this->thriftSQL->query( $this->queryStr );
+    $this->buffer = array();
+    $this->location = 0;
+    $this->thriftSQLQuery->wait();
+  }
+}

src/ThriftSQL/Utils/Iterator.php

@@ -198,6 +198,7 @@
   'ThriftSQL\\TTypeQualifiers' => 'Packages/TCLIService/Types.php',
   'ThriftSQL\\TUnionTypeEntry' => 'Packages/TCLIService/Types.php',
   'ThriftSQL\\TUserDefinedTypeEntry' => 'Packages/TCLIService/Types.php',
+  'ThriftSQL\\Utils\\Iterator' => 'ThriftSQL/Utils/Iterator.php',
   'ThriftSQL\\Utils\\QueryCleaner' => 'ThriftSQL/Utils/QueryCleaner.php',
   'ThriftSQL\\Utils\\Sleeper' => 'ThriftSQL/Utils/Sleeper.php',
   'Thrift\\Base\\TBase' => 'Thrift/Base/TBase.php',