Quick start (#10)

Author: akshaisarma

docs/quick-start/bullet-on-storm-with-rest.md

@@ -26,6 +26,7 @@ The core Bullet logic (a library) that can be used to implement Bullet on differ
 
 |    Date      |                                        Release                                        | Highlights |
 | ------------ | ------------------------------------------------------------------------------------- | ---------- |
+| 2018-06-22   | [**0.4.1**](https://github.com/bullet-db/bullet-core/releases/tag/bullet-core-0.4.1)  | Added RESTPublisher HTTP Timeout Setting |
 | 2018-06-18   | [**0.4.0**](https://github.com/bullet-db/bullet-core/releases/tag/bullet-core-0.4.0)  | Added support for Integer and Float data types, and configurable BulletRecordProvider class used to instantiate BulletRecords in bullet-core |
 | 2018-04-11   | [**0.3.4**](https://github.com/bullet-db/bullet-core/releases/tag/bullet-core-0.3.4)  | Pre-Start delaying and Buffering changes - queries are now buffered at the start of a query instead of start of each window |
 | 2018-03-30   | [**0.3.3**](https://github.com/bullet-db/bullet-core/releases/tag/bullet-core-0.3.3)  | Bug fix for com.yahoo.bullet.core.querying.Querier#isClosedForPartition |

docs/quick-start/storm.md

@@ -2,10 +2,10 @@
 
 set -euo pipefail
 
-BULLET_EXAMPLES_VERSION=0.4.0
-BULLET_UI_VERSION=0.4.0
-BULLET_WS_VERSION=0.1.1
-STORM_VERSION=1.0.3
+BULLET_EXAMPLES_VERSION=0.5.1
+BULLET_UI_VERSION=0.5.0
+BULLET_WS_VERSION=0.2.2
+STORM_VERSION=1.2.2
 NVM_VERSION=0.33.1
 NODE_VERSION=6.9.4
 
@@ -80,7 +80,6 @@ install_storm() {
 
     println "Configuring Storm ..."
     export PATH="${BACKEND}/${STORM}/bin/:${PATH}"
-    echo 'drpc.servers: ["127.0.0.1"]' >> "${BACKEND}/${STORM}/conf/storm.yaml"
     println "Done!"
 }
 
@@ -91,9 +90,6 @@ launch_storm() {
     println "Launching Storm Nimbus..."
     storm nimbus &
 
-    println "Launching Storm DRPC..."
-    storm drpc &
-
     println "Launching Storm UI..."
     storm ui &
 
@@ -112,7 +108,7 @@ launch_storm() {
 
 launch_bullet_storm() {
     println "Copying Bullet topology configuration and artifacts..."
-    cp "${BULLET_EXAMPLES}/storm"/* "${BULLET_HOME}/backend/storm"
+    cp "${BULLET_EXAMPLES}/backend/storm"/* "${BULLET_HOME}/backend/storm"
 
     println "Launching the Bullet topology..."
     println "=============================================================================="
@@ -123,11 +119,6 @@ launch_bullet_storm() {
     println "=============================================================================="
     sleep 30
     println "=============================================================================="
-
-    println "Testing the Storm topology"
-    println ""
-    println "Getting one random record from the Bullet topology..."
-    curl -s -X POST -d '{"id":"", "content":"{}"}' http://localhost:3774/drpc/bullet-query
     println "Done!"
 }
 
@@ -138,16 +129,16 @@ launch_bullet_web_service() {
     println "Downloading Bullet Web Service ${BULLET_WS_VERSION}..."
     download "http://jcenter.bintray.com/com/yahoo/bullet/bullet-service/${BULLET_WS_VERSION}" "${BULLET_WS_JAR}"
 
-    println "Configuring Bullet Web Service and plugging in Storm DRPC PubSub..."
+    println "Configuring Bullet Web Service and plugging in In-Memory REST PubSub..."
     cp "${BULLET_DOWNLOADS}/${BULLET_WS_JAR}" "${BULLET_SERVICE_HOME}/bullet-service.jar"
-    cp "${BULLET_EXAMPLES}/storm"/*jar-with-dependencies.jar "${BULLET_SERVICE_HOME}/bullet-storm-jar-with-dependencies.jar"
     cp "${BULLET_EXAMPLES}/web-service/"example_* "${BULLET_SERVICE_HOME}/"
 
-    println "Launching Bullet Web Service..."
+    println "Launching Bullet Web Service with the built-in REST PubSub enabled..."
     cd "${BULLET_SERVICE_HOME}"
-    java -Dloader.path=bullet-storm-jar-with-dependencies.jar -jar bullet-service.jar \
-         --bullet.pubsub.config=example_drpc_pubsub_config.yaml --bullet.schema.file=example_columns.json \
-         --server.port=9999  --logging.path="${BULLET_SERVICE_HOME}" --logging.file=log.txt &> "${BULLET_SERVICE_HOME}/log.txt" &
+    java -jar ./bullet-service.jar \
+         --bullet.pubsub.config=example_rest_pubsub_config.yaml --bullet.schema.file=example_columns.json \
+         --server.port=9999  --bullet.pubsub.builtin.rest.enabled=true --logging.path="${BULLET_SERVICE_HOME}" \
+         --logging.file=log.txt &> "${BULLET_SERVICE_HOME}/log.txt" &
 
     println "Sleeping for 15 s to ensure Bullet Web Service is up..."
     sleep 15
@@ -160,7 +151,7 @@ launch_bullet_web_service() {
     println "Getting column schema from the Web Service..."
     println ""
     curl -s http://localhost:9999/api/bullet/columns
-    println "Finished Bullet Web Service test"
+    println "Finished Bullet Web Service test!"
 }
 
 install_node() {

docs/releases.md

@@ -99,5 +99,15 @@ bullet.pubsub.kafka.response.topic.name: "bullet.responses"
 ## https://github.com/bullet-db/bullet-core/blob/master/src/main/resources/bullet_defaults.yaml
 ########################################################################################################################
 ########################################################################################################################
+bullet.query.aggregation.raw.max.size: 500
+bullet.query.aggregation.max.size: 1024
+bullet.query.aggregation.count.distinct.sketch.entries: 16384
+bullet.query.aggregation.group.sketch.entries: 1024
+bullet.query.aggregation.distribution.sketch.entries: 1024
+bullet.query.aggregation.distribution.max.points: 200
+bullet.query.aggregation.distribution.generated.points.rounding: 6
+bullet.query.aggregation.top.k.sketch.entries: 1024
+bullet.query.aggregation.top.k.sketch.error.type: "NFN"
+bullet.result.metadata.enable: true
 # Factory class to get new BulletRecords.
 bullet.record.provider.class.name: "com.yahoo.bullet.record.SimpleBulletRecordProvider"

examples/install-all-storm.sh

@@ -3,7 +3,7 @@
 # We pass 20 and 100 to the RandomSpout, which means it generates up to 20 random records every 100 ms.
 storm jar bullet-storm-example-1.0-SNAPSHOT-jar-with-dependencies.jar \
           com.yahoo.bullet.storm.Topology \
-          --bullet-conf bullet_settings.yaml \
+          --bullet-conf ./bullet_settings.yaml \
           --bullet-spout com.yahoo.bullet.storm.examples.RandomSpout \
           --bullet-spout-parallelism 1 \
           --bullet-spout-cpu-load 100.0 \

examples/spark/src/main/resources/bullet_spark_kafka_settings.yaml

@@ -7,7 +7,7 @@
     <packaging>jar</packaging>
     <name>bullet-storm-example</name>
     <scm>
-        <developerConnection>scm:git:ssh://git@github.com/yahoo/bullet-docs.git</developerConnection>
+        <developerConnection>scm:git:ssh://git@github.com/bullet-db/bullet-db.github.io.git</developerConnection>
         <tag>HEAD</tag>
     </scm>
 
@@ -26,7 +26,7 @@
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <maven.compiler.source>1.8</maven.compiler.source>
         <maven.compiler.target>1.8</maven.compiler.target>
-        <bullet.storm.version>0.8.2</bullet.storm.version>
+        <bullet.storm.version>0.8.3</bullet.storm.version>
         <bullet.core.version>0.4.0</bullet.core.version>
         <bullet.record.version>0.2.0</bullet.record.version>
         <storm.version>1.0.3</storm.version>

examples/spark/src/main/resources/bullet_spark_rest_settings.yaml

@@ -1,29 +1,36 @@
-# Custom settings for the example
+# Custom or notable settings for the example
 # Settings not overridden will default to the defaults in bullet_storm_defaults.yaml in the bullet-storm artifact
 # and to the defaults in bullet_defaults.yaml in the bullet-core artifact.
-# Settings that start with bullet.topology are Storm settings and everything else are Bullet Core settings.
+
+# Storm settings
 bullet.topology.name: "bullet"
 bullet.topology.metrics.enable: false
 bullet.topology.metrics.built.in.enable: false
 bullet.topology.query.spout.parallelism: 1
 bullet.topology.query.spout.cpu.load: 30.0
 bullet.topology.query.spout.memory.on.heap.load: 256.0
-bullet.topology.query.spout.memory.off.heap.load: 192.0
+bullet.topology.query.spout.memory.off.heap.load: 160.0
 bullet.topology.filter.bolt.parallelism: 1
 bullet.topology.filter.bolt.cpu.load: 100.0
 bullet.topology.filter.bolt.memory.on.heap.load: 256.0
-bullet.topology.filter.bolt.memory.off.heap.load: 192.0
+bullet.topology.filter.bolt.memory.off.heap.load: 160.0
 bullet.topology.return.bolt.parallelism: 1
 bullet.topology.result.bolt.cpu.load: 10.0
 bullet.topology.result.bolt.memory.on.heap.load: 128.0
-bullet.topology.result.bolt.memory.off.heap.load: 192.0
+bullet.topology.result.bolt.memory.off.heap.load: 160.0
+bullet.topology.loop.bolt.parallelism: 1
+bullet.topology.loop.bolt.cpu.load: 10.0
+bullet.topology.loop.bolt.memory.on.heap.load: 128.0
+bullet.topology.loop.bolt.memory.off.heap.load: 160.0
 bullet.topology.join.bolt.parallelism: 1
 bullet.topology.join.bolt.cpu.load: 20.0
 bullet.topology.join.bolt.memory.on.heap.load: 128.0
-bullet.topology.join.bolt.memory.off.heap.load: 192.0
-bullet.topology.join.bolt.error.tick.timeout: 3
-bullet.topology.join.bolt.query.tick.timeout: 3
-bullet.topology.tick.interval.secs: 1
+bullet.topology.join.bolt.memory.off.heap.load: 160.0
+bullet.topology.join.bolt.query.post.finish.buffer.ticks: 3
+bullet.topology.join.bolt.query.pre.start.delay.ticks: 2
+bullet.topology.tick.spout.interval.ms: 100
+
+# Bullet Core settings
 bullet.query.aggregation.raw.max.size: 500
 bullet.query.aggregation.max.size: 1024
 bullet.query.aggregation.count.distinct.sketch.entries: 16384
@@ -33,47 +40,11 @@ bullet.query.aggregation.distribution.max.points: 200
 bullet.query.aggregation.distribution.generated.points.rounding: 6
 bullet.query.aggregation.top.k.sketch.entries: 1024
 bullet.query.aggregation.top.k.sketch.error.type: "NFN"
-bullet.query.max.duration: 570000
 bullet.result.metadata.enable: true
-bullet.result.metadata.metrics:
-    - name: "Query Identifier"
-      key: "query_id"
-    - name: "Query Body"
-      key: "query"
-    - name: "Query Creation Time"
-      key: "query_receive_time"
-    - name: "Query Termination Time"
-      key: "query_finish_time"
-    - name: "Sketch Metadata"
-      key: "sketches"
-    - name: "Estimated Result"
-      key: "was_estimated"
-    - name: "Standard Deviations"
-      key: "standard_deviations"
-    - name: "Family"
-      key: "family"
-    - name: "Size"
-      key: "size"
-    - name: "Theta"
-      key: "theta"
-    - name: "Uniques Estimate"
-      key: "uniques_estimate"
-    - name: "Minimum Value"
-      key: "minimum_value"
-    - name: "Maximum Value"
-      key: "maximum_value"
-    - name: "Items Seen"
-      key: "items_seen"
-    - name: "Normalized Rank Error"
-      key: "normalized_rank_error"
-    - name: "Maximum Count Error"
-      key: "maximum_count_error"
-    - name: "Active Items"
-      key: "active_items"
-bullet.record.inject.timestamp.enable: true
-bullet.record.inject.timestamp.key: "receive_timestamp"
 
-# Storm DRPC PubSub settings
+# REST PubSub settings
 bullet.pubsub.context.name: "QUERY_PROCESSING"
-bullet.pubsub.class.name: "com.yahoo.bullet.storm.drpc.DRPCPubSub"
-bullet.pubsub.storm.drpc.function: "bullet-query"
+bullet.pubsub.class.name: "com.yahoo.bullet.pubsub.rest.RESTPubSub"
+bullet.pubsub.rest.query.urls:
+    - "http://localhost:9999/api/bullet/pubsub/query"
+bullet.pubsub.rest.result.url: "http://localhost:9999/api/bullet/pubsub/result"

examples/storm/bin/launch.sh

@@ -19,15 +19,14 @@
       "deletions": "query"
     },
     "defaultValues": {
-      "aggregationMaxSize": 512,
-      "rawMaxSize": 100,
-      "durationMaxSecs": 120,
+      "aggregationMaxSize": 1024,
+      "rawMaxSize": 500,
+      "durationMaxSecs": 86400,
       "distributionNumberOfPoints": 11,
       "distributionQuantilePoints": "0, 0.25, 0.5, 0.75, 0.9, 1",
       "distributionQuantileStart": 0,
       "distributionQuantileEnd": 1,
       "distributionQuantileIncrement": 0.1,
-      "queryTimeoutSecs": 3,
       "windowEmitFrequencyMinSecs": 1,
       "everyForRecordBasedWindow": 1,
       "everyForTimeBasedWindow": 2,