Add planck_ecs, replace table with violin graphs

Author: AnneKitsune

Cargo.toml

@@ -7,14 +7,15 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-cgmath = { version = "0.17", feature = ["serde"] }
+bevy_ecs = "0.3"
 bincode = "1.3"
+cgmath = { version = "0.17", feature = ["serde"] }
+hecs = { version = "0.5", features = ["column-serialize", "row-serialize"] }
+legion = "0.3"
+planck_ecs = { version = "1.1.0", features = ["parallel"] }
+rayon = "1.3"
 ron = "0.6"
 serde = { version = "1.0", features = ["derive"] }
-rayon = "1.3"
-legion = "0.3"
-bevy_ecs = "0.3"
-hecs = { version = "0.5", features = ["column-serialize", "row-serialize"] }
 shipyard = "0.4"
 specs = {version = "0.16.1", features = ["serde"] }
 specs-derive = "0.4.1"
@@ -25,4 +26,4 @@ rayon = "1.3"
 
 [[bench]]
 name = "benchmarks"
-harness = false
+harness = false

benches/benchmarks.rs

@@ -15,6 +15,10 @@ fn bench_simple_insert(c: &mut Criterion) {
         let mut bench = hecs::simple_insert::Benchmark::new();
         b.iter(move || bench.run());
     });
+    group.bench_function("planck_ecs", |b| {
+        let mut bench = planck_ecs::simple_insert::Benchmark::new();
+        b.iter(move || bench.run());
+    });
     group.bench_function("shipyard", |b| {
         let mut bench = shipyard::simple_insert::Benchmark::new();
         b.iter(move || bench.run());
@@ -43,6 +47,10 @@ fn bench_simple_iter(c: &mut Criterion) {
         let mut bench = hecs::simple_iter::Benchmark::new();
         b.iter(move || bench.run());
     });
+    group.bench_function("planck_ecs", |b| {
+        let mut bench = planck_ecs::simple_iter::Benchmark::new();
+        b.iter(move || bench.run());
+    });
     group.bench_function("shipyard", |b| {
         let mut bench = shipyard::simple_iter::Benchmark::new();
         b.iter(move || bench.run());
@@ -71,6 +79,10 @@ fn bench_frag_iter_bc(c: &mut Criterion) {
         let mut bench = hecs::frag_iter::Benchmark::new();
         b.iter(move || bench.run());
     });
+    group.bench_function("planck_ecs", |b| {
+        let mut bench = planck_ecs::frag_iter::Benchmark::new();
+        b.iter(move || bench.run());
+    });
     group.bench_function("shipyard", |b| {
         let mut bench = shipyard::frag_iter::Benchmark::new();
         b.iter(move || bench.run());
@@ -95,6 +107,10 @@ fn bench_schedule(c: &mut Criterion) {
         let mut bench = bevy::schedule::Benchmark::new();
         b.iter(move || bench.run());
     });
+    group.bench_function("planck_ecs", |b| {
+        let mut bench = planck_ecs::schedule::Benchmark::new();
+        b.iter(move || bench.run());
+    });
     group.bench_function("shipyard", |b| {
         let mut bench = shipyard::schedule::Benchmark::new();
         b.iter(move || bench.run());
@@ -151,6 +167,10 @@ fn bench_add_remove(c: &mut Criterion) {
         let mut bench = hecs::add_remove::Benchmark::new();
         b.iter(move || bench.run());
     });
+    group.bench_function("planck_ecs", |b| {
+        let mut bench = planck_ecs::add_remove::Benchmark::new();
+        b.iter(move || bench.run());
+    });
     group.bench_function("shipyard", |b| {
         let mut bench = shipyard::add_remove::Benchmark::new();
         b.iter(move || bench.run());

readme.md

@@ -4,21 +4,6 @@ A suite of benchmarks designed to test and compare Rust ECS library performance
 
 The full benchmark report is available [here](https://rust-gamedev.github.io/ecs_bench_suite/target/criterion/report/index.html).
 
-|                  | legion (\*)           | bevy       | hecs       | shipyard (\*)         | specs       |
-|------------------|:----------------------|:----------:|:----------:|:---------------------:|:-----------:|
-| simple_insert    | **383μs**             | 636μs      | 640μs      | 2.08ms                | 1.90ms      |
-| simple_iter      | 13.2μs (**11.2μs**)   | 12.9μs     | **12.0μs** | 86.3μs (24.2μs)       | 28.8ms      |
-| frag_iter        | 441ns                 | 554ns      | 452ns      | **121ns**             | 1.41μs      |
-| heavy_compute    | **686μs** (687μs)     | 958μs      | 972μs      | **693μs** (693μs)     | 968μs       |
-| schedule         | **54.3μs** (53.7μs)   | 80.3μs     | -          | 372μs (132μs)         | 155μs       |
-| add_remove       | 4.45ms                | 6.71ms     | 7.86ms     | 237μs                 | **123μs**   |
-| serialize_text   | **12.5ms**            | -          | -          | -                     | -           |
-| serialize_binary | **6.50ms**            | -          | -          | -                     | -           |
-
-(*): The values in parentheses are results where per-benchmark storage optimizations were applied. Some of these are mutually exclusive, so with and without "packing" typically represent best and worst-case performance for the ECS.
-
-The best result for each benchmark is marked in bold text. Note that run to run variance for these benchmarks is typically 2-3%, with outliers as much as 10%. All micro-benchmarks should be taken with a grain of salt, and any benchmarks within a few percent of each other should be considered "effectively equal".
-
 ## The Benchmarks
 
 ### Simple Insert
@@ -27,6 +12,8 @@ This benchmark is designed to test the base cost of constructing entities and mo
 
 Inserts 10,000 entities, each with 4 components: `Transform(mat4x4)`, `Position(vec3)`, `Rotation(vec3)` and `Velocity(vec3)`.
 
+![](./target/criterion/simple_insert/report/violin.svg)
+
 ### Simple Iter
 
 This benchmark is designed to test the core overheads involved in component iteration in best-case conditions. The iteration should occur on a single CPU core.
@@ -35,6 +22,8 @@ Dataset: 10,000 entities, each with 4 components: `Transform(mat4x4)`, `Position
 
 Test: Iterate through all entities with `Position` and `Velocity`, and add velocity onto position.
 
+![](./target/criterion/simple_iter/report/violin.svg)
+
 ### Fragmented Iter
 
 This benchmark is designed to test how the ECS handles iteration through a fragmented dataset. The iteration should occur on a single CPU core.
@@ -43,6 +32,8 @@ Dataset: 26 component types (`A(f32)` through `Z(f32)`), each with 20 entities p
 
 Test: Iterate through all entities with a `Data` component and double its value.
 
+![](./target/criterion/fragmented_iter/report/violin.svg)
+
 ### System Scheduling
 
 This benchmark is designed to test how efficiently the ECS can schedule multiple independent systems on a multi-core CPU. This is primarily an outer-parallelism test. Each system should execute on a single CPU core.
@@ -62,6 +53,8 @@ Three systems accessing the following components mutably, where each system swap
 * `(C, D)`
 * `(C, E)`
 
+![](./target/criterion/schedule/report/violin.svg)
+
 ### Heavy Compute
 
 This benchmark is designed to test the ECS's ability to scale when it is allowed to run a system over multiple CPU cores. This is primarily an inner-parallelism test.
@@ -70,6 +63,8 @@ Dataset: 10,000 entities with a `mat4x4` component.
 
 Test: Iterate through all `mat4x4` components, and invert the matrix 10 times.
 
+![](./target/criterion/heavy_compute/report/violin.svg)
+
 ### Add/Remove Component
 
 This benchmark is designed to test how quickly the ECS can add and then remove a component from an existing entity.
@@ -78,10 +73,15 @@ Dataset: 1,000 entities with a single `A` component.
 
 Test: Iterate through all entities, adding a `B` component. Then iterate through all entities again, removing their `B` component.
 
+![](./target/criterion/add_remove_component/report/violin.svg)
+
 ### Serialize
 
 This benchmark is designed to test how quickly the ECS and serialize and deserialize its entities in both text (RON) and binary (bincode) formats.
 
 Dataset: 1000 entities with `Transform(mat4x4)`, `Position(vec3)`, `Rotation(vec3)` and `Velocity(vec3)` components.
 
 Test: Serialize all entities to RON and bincode formats in-memory. Then deserialize back into the ECS. The RON and bincode formats should be separate benchmark tests.
+
+![](./target/criterion/serialize_text/report/violin.svg)
+![](./target/criterion/serialize_binary/report/violin.svg)

src/lib.rs

@@ -4,6 +4,7 @@ pub mod bevy;
 pub mod hecs;
 pub mod legion;
 pub mod legion_packed;
+pub mod planck_ecs;
 pub mod shipyard;
 pub mod shipyard_packed;
 pub mod specs;

src/planck_ecs/add_remove.rs

@@ -0,0 +1,33 @@
+use planck_ecs::*;
+struct A(f32);
+struct B(f32);
+
+pub struct Benchmark(Vec<Entity>, Components<A>, Components<B>);
+
+impl Benchmark {
+    pub fn new() -> Self {
+        let mut entities = Entities::default();
+        let mut comp1 = Components::<A>::default();
+        let comp2 = Components::<B>::default();
+
+        let entities = (0..10000)
+            .map(|_| {
+                let e = entities.create();
+                comp1.insert(e, A(0.0));
+                e
+            })
+            .collect();
+        Self(entities, comp1, comp2)
+    }
+
+    pub fn run(&mut self) {
+        let b_storage = &mut self.2;
+        for entity in &self.0 {
+            b_storage.insert(*entity, B(0.0));
+        }
+
+        for entity in &self.0 {
+            b_storage.remove(*entity);
+        }
+    }
+}

src/planck_ecs/frag_iter.rs

@@ -0,0 +1,41 @@
+use planck_ecs::*;
+
+macro_rules! create_entities {
+    ($world:ident; $( $variants:ident ),*) => {
+        $(
+            struct $variants(f32);
+            $world.initialize::<Components<$variants>>();
+            (0..20)
+            .for_each(|_| {
+                let e = $world.get_mut::<Entities>().unwrap().create();
+                $world.get_mut::<Components<_>>().unwrap().insert(e, $variants(0.0));
+                $world.get_mut::<Components<_>>().unwrap().insert(e, Data(1.0));
+            });
+        )*
+    };
+}
+struct Data(f32);
+
+fn frag_iter_system(data_storage: &mut Components<Data>) -> SystemResult {
+    for data in join!(&mut data_storage) {
+        data.0 *= 2.0;
+    }
+    Ok(())
+}
+
+pub struct Benchmark(World, System);
+
+impl Benchmark {
+    pub fn new() -> Self {
+        let mut world = World::default();
+        world.initialize::<Entities>();
+        world.initialize::<Components<Data>>();
+        create_entities!(world; A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y, Z);
+
+        Self(world, frag_iter_system.system())
+    }
+
+    pub fn run(&mut self) {
+        self.1.run(&self.0).unwrap();
+    }
+}

src/planck_ecs/mod.rs

@@ -0,0 +1,7 @@
+pub mod add_remove;
+pub mod frag_iter;
+// We don't have inner parallelism, only outer.
+//pub mod heavy_compute;
+pub mod schedule;
+pub mod simple_insert;
+pub mod simple_iter;

src/planck_ecs/schedule.rs

@@ -0,0 +1,82 @@
+use planck_ecs::*;
+struct A(f32);
+struct B(f32);
+struct C(f32);
+struct D(f32);
+struct E(f32);
+
+fn ab_system(a_store: &mut Components<A>, b_store: &mut Components<B>) -> SystemResult {
+    for (a, b) in join!(&mut a_store && &mut b_store) {
+        std::mem::swap(&mut a.unwrap().0, &mut b.unwrap().0);
+    }
+    Ok(())
+}
+
+fn cd_system(c_store: &mut Components<C>, d_store: &mut Components<D>) -> SystemResult {
+    for (c, d) in join!(&mut c_store && &mut d_store) {
+        std::mem::swap(&mut c.unwrap().0, &mut d.unwrap().0);
+    }
+    Ok(())
+}
+
+fn ce_system(c_store: &mut Components<C>, e_store: &mut Components<E>) -> SystemResult {
+    for (c, e) in join!(&mut c_store && &mut e_store) {
+        std::mem::swap(&mut c.unwrap().0, &mut e.unwrap().0);
+    }
+    Ok(())
+}
+
+pub struct Benchmark(World, Dispatcher);
+
+impl Benchmark {
+    pub fn new() -> Self {
+        let mut world = World::default();
+        world.initialize::<Entities>();
+        world.initialize::<Components<A>>();
+        world.initialize::<Components<B>>();
+        world.initialize::<Components<C>>();
+        world.initialize::<Components<D>>();
+        world.initialize::<Components<E>>();
+        (0..10000).for_each(|_| {
+            let e = world.get_mut::<Entities>().unwrap().create();
+            world.get_mut::<Components<_>>().unwrap().insert(e, A(0.0));
+        });
+        (0..10000).for_each(|_| {
+            let e = world.get_mut::<Entities>().unwrap().create();
+            world.get_mut::<Components<_>>().unwrap().insert(e, A(0.0));
+            world.get_mut::<Components<_>>().unwrap().insert(e, B(0.0));
+        });
+        (0..10000).for_each(|_| {
+            let e = world.get_mut::<Entities>().unwrap().create();
+            world.get_mut::<Components<_>>().unwrap().insert(e, A(0.0));
+            world.get_mut::<Components<_>>().unwrap().insert(e, B(0.0));
+            world.get_mut::<Components<_>>().unwrap().insert(e, C(0.0));
+        });
+        (0..10000).for_each(|_| {
+            let e = world.get_mut::<Entities>().unwrap().create();
+            world.get_mut::<Components<_>>().unwrap().insert(e, A(0.0));
+            world.get_mut::<Components<_>>().unwrap().insert(e, B(0.0));
+            world.get_mut::<Components<_>>().unwrap().insert(e, C(0.0));
+            world.get_mut::<Components<_>>().unwrap().insert(e, D(0.0));
+        });
+        (0..10000).for_each(|_| {
+            let e = world.get_mut::<Entities>().unwrap().create();
+            world.get_mut::<Components<_>>().unwrap().insert(e, A(0.0));
+            world.get_mut::<Components<_>>().unwrap().insert(e, B(0.0));
+            world.get_mut::<Components<_>>().unwrap().insert(e, C(0.0));
+            world.get_mut::<Components<_>>().unwrap().insert(e, E(0.0));
+        });
+
+        let dispatcher = DispatcherBuilder::new()
+            .add(ab_system)
+            .add(cd_system)
+            .add(ce_system)
+            .build(&mut world);
+
+        Self(world, dispatcher)
+    }
+
+    pub fn run(&mut self) {
+        self.1.run_par(&self.0).unwrap();
+    }
+}

src/planck_ecs/simple_insert.rs

@@ -0,0 +1,41 @@
+use cgmath::*;
+use planck_ecs::*;
+
+#[derive(Copy, Clone)]
+struct Transform(Matrix4<f32>);
+#[derive(Copy, Clone)]
+struct Position(Vector3<f32>);
+#[derive(Copy, Clone)]
+struct Rotation(Vector3<f32>);
+#[derive(Copy, Clone)]
+struct Velocity(Vector3<f32>);
+
+pub struct Benchmark;
+
+impl Benchmark {
+    pub fn new() -> Self {
+        Self
+    }
+
+    pub fn run(&mut self) {
+        let mut entities = Entities::default();
+        let mut comp1 = Components::<Transform>::default();
+        let mut comp2 = Components::<Position>::default();
+        let mut comp3 = Components::<Rotation>::default();
+        let mut comp4 = Components::<Velocity>::default();
+
+        let en = (0..10000).map(|_| entities.create()).collect::<Vec<_>>();
+        en.iter().for_each(|e| {comp1.insert(*e, Transform(Matrix4::<f32>::from_scale(1.0)));});
+        en.iter().for_each(|e| {comp2.insert(*e, Position(Vector3::unit_x()));});
+        en.iter().for_each(|e| {comp3.insert(*e, Rotation(Vector3::unit_x()));});
+        en.iter().for_each(|e| {comp4.insert(*e, Velocity(Vector3::unit_x()));});
+
+        /*(0..10000).for_each(|_| {
+            let e = entities.create();
+            comp1.insert(e, Transform(Matrix4::<f32>::from_scale(1.0)));
+            comp2.insert(e, Position(Vector3::unit_x()));
+            comp3.insert(e, Rotation(Vector3::unit_x()));
+            comp4.insert(e, Velocity(Vector3::unit_x()));
+        });*/
+    }
+}