add insert benchmarks comparing myxql/postgrex/exqlite (#111)

Author: kevinlang

bench/README.md

@@ -0,0 +1,36 @@
+# Ecto Benchmarks
+
+Ecto has a benchmark suite to track performance of sensitive operations. Benchmarks
+are run using the [Benchee](https://github.com/PragTob/benchee) library and
+need PostgreSQL and MySQL up and running.
+
+To run the benchmarks tests just type in the console:
+
+```
+# POSIX-compatible shells
+$ MIX_ENV=bench mix run bench/bench_helper.exs
+```
+
+```
+# other shells
+$ env MIX_ENV=bench mix run bench/bench_helper.exs
+```
+
+Benchmarks are inside the `scripts/` directory and are divided into two
+categories:
+
+* `micro benchmarks`: Operations that don't actually interface with the database,
+but might need it up and running to start the Ecto agents and processes.
+
+* `macro benchmarks`: Operations that are actually run in the database. This are
+more likely to integration tests.
+
+You can also run a benchmark individually by giving the path to the benchmark
+script instead of `bench/bench_helper.exs`.
+
+# Docker
+I had Postgres already installed and running locally, but needed to get MySQL up and running. The easiest way to do this is with this command:
+
+```
+docker run -p 3306:3306 --name mysql_server -e MYSQL_ALLOW_EMPTY_PASSWORD=yes mysql:5.7
+```

bench/bench_helper.exs

@@ -0,0 +1,7 @@
+# Micro benchmarks
+Code.require_file("scripts/micro/load_bench.exs", __DIR__)
+Code.require_file("scripts/micro/to_sql_bench.exs", __DIR__)
+
+## Macro benchmarks needs postgresql and mysql up and running
+Code.require_file("scripts/macro/insert_bench.exs", __DIR__)
+Code.require_file("scripts/macro/all_bench.exs", __DIR__)

bench/scripts/macro/all_bench.exs

@@ -0,0 +1,53 @@
+# -----------------------------------Goal--------------------------------------
+# Compare the performance of querying all objects of the different supported
+# databases
+
+# -------------------------------Description-----------------------------------
+# This benchmark tracks performance of querying a set of objects registered in
+# the database with Repo.all/2 function. The query pass through
+# the steps of translating the SQL statements, sending them to the database and
+# load the results into Ecto structures. Both, Ecto Adapters and Database itself
+# play a role and can affect the results of this benchmark.
+
+# ----------------------------Factors(don't change)---------------------------
+# Different adapters supported by Ecto with the proper database up and running
+
+# ----------------------------Parameters(change)-------------------------------
+# There is only a unique parameter in this benchmark, the User objects to be
+# fetched.
+
+Code.require_file("../../support/setup.exs", __DIR__)
+
+alias Ecto.Bench.User
+
+limit = 5_000
+
+users =
+  1..limit
+  |> Enum.map(fn _ -> User.sample_data() end)
+
+# We need to insert data to fetch
+Ecto.Bench.PgRepo.insert_all(User, users)
+Ecto.Bench.MyXQLRepo.insert_all(User, users)
+
+jobs = %{
+  "Pg Repo.all/2" => fn -> Ecto.Bench.PgRepo.all(User, limit: limit) end,
+  "MyXQL Repo.all/2" => fn -> Ecto.Bench.MyXQLRepo.all(User, limit: limit) end
+}
+
+path = System.get_env("BENCHMARKS_OUTPUT_PATH") || "bench/results"
+file = Path.join(path, "all.json")
+
+Benchee.run(
+  jobs,
+  formatters: [Benchee.Formatters.JSON, Benchee.Formatters.Console],
+  formatter_options: [json: [file: file]],
+  time: 10,
+  after_each: fn results ->
+    ^limit = length(results)
+  end
+)
+
+# Clean inserted data
+Ecto.Bench.PgRepo.delete_all(User)
+Ecto.Bench.MyXQLRepo.delete_all(User)

bench/scripts/macro/insert_bench.exs

@@ -0,0 +1,46 @@
+# -----------------------------------Goal--------------------------------------
+# Compare the performance of inserting changesets and structs in the different
+# supported databases
+
+# -------------------------------Description-----------------------------------
+# This benchmark tracks performance of inserting changesets and structs in the
+# database with Repo.insert!/1 function. The query pass through
+# the steps of translating the SQL statements, sending them to the database and
+# returning the result of the transaction. Both, Ecto Adapters and Database itself
+# play a role and can affect the results of this benchmark.
+
+# ----------------------------Factors(don't change)---------------------------
+# Different adapters supported by Ecto with the proper database up and running
+
+# ----------------------------Parameters(change)-------------------------------
+# Different inputs to be inserted, aka Changesets and Structs
+
+Code.require_file("../../support/setup.exs", __DIR__)
+
+alias Ecto.Bench.User
+
+inputs = %{
+  "Struct" => struct(User, User.sample_data()),
+  "Changeset" => User.changeset(User.sample_data())
+}
+
+jobs = %{
+  "Exqlite Insert" => fn entry -> Ecto.Bench.ExqliteRepo.insert!(entry) end,
+  "Pg Insert" => fn entry -> Ecto.Bench.PgRepo.insert!(entry) end,
+  "MyXQL Insert" => fn entry -> Ecto.Bench.MyXQLRepo.insert!(entry) end
+}
+
+path = System.get_env("BENCHMARKS_OUTPUT_PATH") || "bench/results"
+file = Path.join(path, "insert.json")
+
+Benchee.run(
+  jobs,
+  inputs: inputs,
+  formatters: [Benchee.Formatters.JSON, Benchee.Formatters.Console],
+  formatter_options: [json: [file: file]]
+)
+
+# Clean inserted data
+Ecto.Bench.ExqliteRepo.delete_all(User)
+Ecto.Bench.PgRepo.delete_all(User)
+Ecto.Bench.MyXQLRepo.delete_all(User)

bench/scripts/micro/load_bench.exs

@@ -0,0 +1,55 @@
+# -----------------------------------Goal--------------------------------------
+# Compare the implementation of loading raw database data into Ecto structures by
+# the different database adapters
+
+# -------------------------------Description-----------------------------------
+# Repo.load/2 is an important step of a database query.
+# This benchmark tracks performance of loading "raw" data into ecto structures
+# Raw data can be in different types (e.g. keyword lists, maps), in this tests
+# we benchmark against map inputs
+
+# ----------------------------Factors(don't change)---------------------------
+# Different adapters supported by Ecto, each one has its own implementation that
+# is tested against different inputs
+
+# ----------------------------Parameters(change)-------------------------------
+# Different sizes of raw data(small, medium, big) and different attribute types
+# such as UUID, Date and Time fetched from the database and needs to be
+# loaded into Ecto structures.
+
+Code.require_file("../../support/setup.exs", __DIR__)
+
+alias Ecto.Bench.User
+
+inputs = %{
+  "Small 1 Thousand" =>
+    1..1_000 |> Enum.map(fn _ -> %{name: "Alice", email: "email@email.com"} end),
+  "Medium 100 Thousand" =>
+    1..100_000 |> Enum.map(fn _ -> %{name: "Alice", email: "email@email.com"} end),
+  "Big 1 Million" =>
+    1..1_000_000 |> Enum.map(fn _ -> %{name: "Alice", email: "email@email.com"} end),
+  "Time attr" =>
+    1..100_000 |> Enum.map(fn _ -> %{name: "Alice", time_attr: ~T[21:25:04.361140]} end),
+  "Date attr" => 1..100_000 |> Enum.map(fn _ -> %{name: "Alice", date_attr: ~D[2018-06-20]} end),
+  "NaiveDateTime attr" =>
+    1..100_000
+    |> Enum.map(fn _ -> %{name: "Alice", naive_datetime_attr: ~N[2019-06-20 21:32:07.424178]} end),
+  "UUID attr" =>
+    1..100_000
+    |> Enum.map(fn _ -> %{name: "Alice", uuid: Ecto.UUID.bingenerate()} end)
+}
+
+jobs = %{
+  "Pg Loader" => fn data -> Enum.map(data, &Ecto.Bench.PgRepo.load(User, &1)) end,
+  "MyXQL Loader" => fn data -> Enum.map(data, &Ecto.Bench.MyXQLRepo.load(User, &1)) end
+}
+
+path = System.get_env("BENCHMARKS_OUTPUT_PATH") || "bench/results"
+file = Path.join(path, "load.json")
+
+Benchee.run(
+  jobs,
+  inputs: inputs,
+  formatters: [Benchee.Formatters.JSON, Benchee.Formatters.Console],
+  formatter_options: [json: [file: file]]
+)

bench/scripts/micro/to_sql_bench.exs

@@ -0,0 +1,64 @@
+# -----------------------------------Goal--------------------------------------
+# Compare the implementation of parsing Ecto.Query objects into SQL queries by
+# the different database adapters
+
+# -------------------------------Description-----------------------------------
+# Repo.to_sql/2 is an important step of a database query.
+# This benchmark tracks performance of parsing Ecto.Query structures into
+# "raw" SQL query strings.
+# Different Ecto.Query objects has multiple combinations and some different attributes
+# depending on the query type. In this tests we benchmark against different
+# query types and complexity.
+
+# ----------------------------Factors(don't change)---------------------------
+# Different adapters supported by Ecto, each one has its own implementation that
+# is tested against different query inputs
+
+# ----------------------------Parameters(change)-------------------------------
+# Different query objects (select, delete, update) to be translated into pure SQL
+# strings.
+
+Code.require_file("../../support/setup.exs", __DIR__)
+
+import Ecto.Query
+
+alias Ecto.Bench.{User, Game}
+
+inputs = %{
+  "Ordinary Select All" => {:all, from(User)},
+  "Ordinary Delete All" => {:delete_all, from(User)},
+  "Ordinary Update All" => {:update_all, from(User, update: [set: [name: "Thor"]])},
+  "Ordinary Where" => {:all, from(User, where: [name: "Thanos", email: "blah@blah"])},
+  "Fetch First Registry" => {:all, first(User)},
+  "Fetch Last Registry" => {:all, last(User)},
+  "Ordinary Order By" => {:all, order_by(User, desc: :name)},
+  "Complex Query 2 Joins" =>
+    {:all,
+     from(User, where: [name: "Thanos"])
+     |> join(:left, [u], ux in User, on: u.id == ux.id)
+     |> join(:right, [j], uj in User, on: j.id == 1 and j.email == "email@email")
+     |> select([u, ux], {u.name, ux.email})},
+  "Complex Query 4 Joins" =>
+    {:all,
+     from(User)
+     |> join(:left, [u], g in Game, on: g.name == u.name)
+     |> join(:right, [g], u in User, on: g.id == 1 and u.email == "email@email")
+     |> join(:inner, [u], g in fragment("SELECT * from games where game.id = ?", u.id))
+     |> join(:left, [g], u in fragment("SELECT * from users = ?", g.id))
+     |> select([u, g], {u.name, g.price})}
+}
+
+jobs = %{
+  "Pg Query Builder" => fn {type, query} -> Ecto.Bench.PgRepo.to_sql(type, query) end,
+  "MyXQL Query Builder" => fn {type, query} -> Ecto.Bench.MyXQLRepo.to_sql(type, query) end
+}
+
+path = System.get_env("BENCHMARKS_OUTPUT_PATH") || "bench/results"
+file = Path.join(path, "to_sql.json")
+
+Benchee.run(
+  jobs,
+  inputs: inputs,
+  formatters: [Benchee.Formatters.JSON, Benchee.Formatters.Console],
+  formatter_options: [json: [file: file]]
+)

bench/support/migrations.exs

@@ -0,0 +1,15 @@
+defmodule Ecto.Bench.CreateUser do
+  use Ecto.Migration
+
+  def change do
+    create table(:users) do
+      add(:name, :string)
+      add(:email, :string)
+      add(:password, :string)
+      add(:time_attr, :time)
+      add(:date_attr, :date)
+      add(:naive_datetime_attr, :naive_datetime)
+      add(:uuid, :binary_id)
+    end
+  end
+end

bench/support/repo.exs

@@ -0,0 +1,43 @@
+pg_bench_url = System.get_env("PG_URL") || "postgres:postgres@localhost"
+myxql_bench_url = System.get_env("MYXQL_URL") || "root@localhost"
+
+Application.put_env(
+  :ecto_sql,
+  Ecto.Bench.PgRepo,
+  url: "ecto://" <> pg_bench_url <> "/ecto_test",
+  adapter: Ecto.Adapters.Postgres,
+  show_sensitive_data_on_connection_error: true
+)
+
+Application.put_env(
+  :ecto_sql,
+  Ecto.Bench.MyXQLRepo,
+  url: "ecto://" <> myxql_bench_url <> "/ecto_test_myxql",
+  adapter: Ecto.Adapters.MyXQL,
+  protocol: :tcp,
+  show_sensitive_data_on_connection_error: true
+)
+
+Application.put_env(
+  :ecto_sql,
+  Ecto.Bench.ExqliteRepo,
+  adapter: Ecto.Adapters.Exqlite,
+  database: "/tmp/exqlite_bench.db",
+  journal_mode: :wal,
+  cache_size: -64000,
+  temp_store: :memory,
+  pool_size: 5,
+  show_sensitive_data_on_connection_error: true
+)
+
+defmodule Ecto.Bench.PgRepo do
+  use Ecto.Repo, otp_app: :ecto_sql, adapter: Ecto.Adapters.Postgres, log: false
+end
+
+defmodule Ecto.Bench.MyXQLRepo do
+  use Ecto.Repo, otp_app: :ecto_sql, adapter: Ecto.Adapters.MyXQL, log: false
+end
+
+defmodule Ecto.Bench.ExqliteRepo do
+  use Ecto.Repo, otp_app: :ecto_sql, adapter: Ecto.Adapters.Exqlite, log: false
+end

bench/support/schemas.exs

@@ -0,0 +1,52 @@
+defmodule Ecto.Bench.User do
+  use Ecto.Schema
+
+  schema "users" do
+    field(:name, :string)
+    field(:email, :string)
+    field(:password, :string)
+    field(:time_attr, :time)
+    field(:date_attr, :date)
+    field(:naive_datetime_attr, :naive_datetime)
+    field(:uuid, :binary_id)
+  end
+
+  @required_attrs [
+    :name,
+    :email,
+    :password,
+    :time_attr,
+    :date_attr,
+    :naive_datetime_attr,
+    :uuid
+  ]
+
+  def changeset() do
+    changeset(sample_data())
+  end
+
+  def changeset(data) do
+    Ecto.Changeset.cast(%__MODULE__{}, data, @required_attrs)
+  end
+
+  def sample_data do
+    %{
+      name: "Lorem ipsum dolor sit amet, consectetur adipiscing elit.",
+      email: "foobar@email.com",
+      password: "mypass",
+      time_attr: Time.utc_now() |> Time.truncate(:second),
+      date_attr: Date.utc_today(),
+      naive_datetime_attr: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second),
+      uuid: Ecto.UUID.generate()
+    }
+  end
+end
+
+defmodule Ecto.Bench.Game do
+  use Ecto.Schema
+
+  schema "games" do
+    field(:name, :string)
+    field(:price, :float)
+  end
+end