From e9a932fc499734591b682c8d3ad2cef83ed09198 Mon Sep 17 00:00:00 2001 From: Andrew Mundy Date: Sat, 4 Jun 2016 13:02:05 +0100 Subject: [PATCH 01/10] WIP CFFI Wrapper for routing table tools - [x] CFFI wrapper for m-Trie - [x] CFFI wrapper for Ordered Covering - [x] Utilities for converting to/from C format - [ ] Scripts to replace desktop/* executables - [ ] Travis updates - [ ] Ignore coverage on cffi_compile.py - [ ] Utilities for writing/reading routing table files - [ ] CFFI wrapper for remove_default_routes when merged For a later PR - SpiNNaker interface for routing table minimisation --- .coveragerc | 3 + .gitignore | 9 ++- .travis.yml | 31 +++++++-- {tests => c_tests}/Makefile | 0 {tests => c_tests}/test_aliases.c | 0 {tests => c_tests}/test_bitset.c | 0 {tests => c_tests}/test_merge.c | 0 {tests => c_tests}/test_mtrie.c | 0 {tests => c_tests}/test_ordered_covering.c | 0 {tests => c_tests}/test_routing_table.c | 0 {tests => c_tests}/tests.c | 0 {tests => c_tests}/tests.h | 0 desktop/ordered_covering.c | 23 +------ include/ordered_covering.h | 22 +++++++ include/routing_table.h | 1 - py_tests/test_mtrie.py | 58 ++++++++++++++++ py_tests/test_ordered_covering.py | 65 ++++++++++++++++++ py_tests/test_utils.py | 44 +++++++++++++ requirements-test.txt | 3 + rig_routing_tables/__init__.py | 4 ++ rig_routing_tables/cffi_compile.py | 55 ++++++++++++++++ rig_routing_tables/cffi_utils.c | 33 ++++++++++ rig_routing_tables/cffi_utils.h | 5 ++ rig_routing_tables/mtrie.py | 42 ++++++++++++ rig_routing_tables/ordered_covering.py | 43 ++++++++++++ rig_routing_tables/utils.py | 77 ++++++++++++++++++++++ rig_routing_tables/version.py | 1 + setup.cfg | 2 + setup.py | 50 ++++++++++++++ 29 files changed, 541 insertions(+), 30 deletions(-) create mode 100644 .coveragerc rename {tests => c_tests}/Makefile (100%) rename {tests => c_tests}/test_aliases.c (100%) rename {tests => c_tests}/test_bitset.c (100%) rename {tests => c_tests}/test_merge.c (100%) rename {tests => c_tests}/test_mtrie.c (100%) rename {tests => c_tests}/test_ordered_covering.c (100%) rename {tests => c_tests}/test_routing_table.c (100%) rename {tests => c_tests}/tests.c (100%) rename {tests => c_tests}/tests.h (100%) create mode 100644 py_tests/test_mtrie.py create mode 100644 py_tests/test_ordered_covering.py create mode 100644 py_tests/test_utils.py create mode 100644 requirements-test.txt create mode 100644 rig_routing_tables/__init__.py create mode 100644 rig_routing_tables/cffi_compile.py create mode 100644 rig_routing_tables/cffi_utils.c create mode 100644 rig_routing_tables/cffi_utils.h create mode 100644 rig_routing_tables/mtrie.py create mode 100644 rig_routing_tables/ordered_covering.py create mode 100644 rig_routing_tables/utils.py create mode 100644 rig_routing_tables/version.py create mode 100644 setup.cfg create mode 100644 setup.py diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..786c033 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,3 @@ +[run] +branch = True +omit = rig_routing_tables/cffi_compile.py diff --git a/.gitignore b/.gitignore index da7edd1..6943c09 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,10 @@ -tests/tests +*.pyc +.eggs/ +.coverage +.cache/* +build/* +*.egg-info/* +c_tests/tests *.o *.gcov *.gcda @@ -7,3 +13,4 @@ tests/tests *.aplx *.elf spinnaker/sark_build.c +_rig_routing_tables.* diff --git a/.travis.yml b/.travis.yml index d728e09..645ba75 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,18 +1,37 @@ -sudo: false -language: c +# Travis Configuration based on that used for rig_c_sa +sudo: false # Use the container based system +language: python # Build under Python to avoid needing to compile Numpy +python: + - 2.7 + - 3.4 + - 3.5 +# Install packages for building and running the test suite addons: apt: packages: - check - valgrind + - libffi-dev + +install: + - CFLAGS=--std=gnu99 python setup.py develop + - pip install -r requirements-test.txt script: - - cd ./tests - - make coverage + # Run the C tests first + - cd c_tests + - make coverage + - cd - + # Then check that the package can be built, and run the Python tests + - > + py.test py_tests \ + --cov rig_routing_tables \ + --cov py_tests + - flake8 rig_routing_tables py_tests after_success: - - bash <(curl -s https://codecov.io/bash) + - bash <(curl -s https://codecov.io/bash) notifications: - email: false + email: false diff --git a/tests/Makefile b/c_tests/Makefile similarity index 100% rename from tests/Makefile rename to c_tests/Makefile diff --git a/tests/test_aliases.c b/c_tests/test_aliases.c similarity index 100% rename from tests/test_aliases.c rename to c_tests/test_aliases.c diff --git a/tests/test_bitset.c b/c_tests/test_bitset.c similarity index 100% rename from tests/test_bitset.c rename to c_tests/test_bitset.c diff --git a/tests/test_merge.c b/c_tests/test_merge.c similarity index 100% rename from tests/test_merge.c rename to c_tests/test_merge.c diff --git a/tests/test_mtrie.c b/c_tests/test_mtrie.c similarity index 100% rename from tests/test_mtrie.c rename to c_tests/test_mtrie.c diff --git a/tests/test_ordered_covering.c b/c_tests/test_ordered_covering.c similarity index 100% rename from tests/test_ordered_covering.c rename to c_tests/test_ordered_covering.c diff --git a/tests/test_routing_table.c b/c_tests/test_routing_table.c similarity index 100% rename from tests/test_routing_table.c rename to c_tests/test_routing_table.c diff --git a/tests/tests.c b/c_tests/tests.c similarity index 100% rename from tests/tests.c rename to c_tests/tests.c diff --git a/tests/tests.h b/c_tests/tests.h similarity index 100% rename from tests/tests.h rename to c_tests/tests.h diff --git a/desktop/ordered_covering.c b/desktop/ordered_covering.c index c1a4970..f66df98 100644 --- a/desktop/ordered_covering.c +++ b/desktop/ordered_covering.c @@ -43,27 +43,6 @@ int entry_cmp(const void *a, const void *b) } -void minimise(table_t *table, unsigned int target_length) -{ - // Create an empty aliases table - aliases_t aliases = aliases_init(); - - // Minimise - oc_minimise(table, target_length, &aliases); - - // Tidy up the aliases table - for (unsigned int i = 0; i < table->size; i++) - { - keymask_t km = table->entries[i].keymask; - if (aliases_contains(&aliases, km)) - { - alias_list_delete((alias_list_t *) aliases_find(&aliases, km)); - aliases_remove(&aliases, km); - } - } -} - - int main(int argc, char *argv[]) { // Usage: @@ -128,7 +107,7 @@ int main(int argc, char *argv[]) qsort(table.entries, table.size, sizeof(entry_t), entry_cmp); // Perform the minimisation - minimise(&table, target_length); + oc_minimise_na(&table, target_length); printf("%u\n", table.size); diff --git a/include/ordered_covering.h b/include/ordered_covering.h index ac256d1..4de6d10 100644 --- a/include/ordered_covering.h +++ b/include/ordered_covering.h @@ -522,6 +522,28 @@ static inline void oc_minimise( } } +// Apply the ordered covering algorithm to a routing table without a +// pre-existing aliases table. +static inline void oc_minimise_na(table_t *table, unsigned int target_length) +{ + // Create an empty aliases table + aliases_t aliases = aliases_init(); + + // Minimise + oc_minimise(table, target_length, &aliases); + + // Tidy up the aliases table + for (unsigned int i = 0; i < table->size; i++) + { + keymask_t km = table->entries[i].keymask; + if (aliases_contains(&aliases, km)) + { + alias_list_delete((alias_list_t *) aliases_find(&aliases, km)); + aliases_remove(&aliases, km); + } + } +} + #define __ORDERED_COVERING_H__ #endif // __ORDERED_COVERING_H__ diff --git a/include/routing_table.h b/include/routing_table.h index 2d34785..d093421 100644 --- a/include/routing_table.h +++ b/include/routing_table.h @@ -58,6 +58,5 @@ typedef struct _table_t entry_t *entries; // Entries in the table } table_t; - #define __ROUTING_TABLE_H__ #endif // __ROUTING_TABLE_H__ diff --git a/py_tests/test_mtrie.py b/py_tests/test_mtrie.py new file mode 100644 index 0000000..c41c3dc --- /dev/null +++ b/py_tests/test_mtrie.py @@ -0,0 +1,58 @@ +import pytest +from rig.routing_table import RoutingTableEntry as RTE +from rig.routing_table import Routes, MinimisationFailedError +from rig_routing_tables.mtrie import minimise + + +def test_minimise_simple(): + """Test minimisation of a simple routing table, we assume the C tests are + extensive! + """ + # Minimise the table: + # N -> 0000 -> S SW + # NE -> 0001 -> S SW + # ? -> 001X -> N + table = [ + RTE({Routes.south, Routes.south_west}, 0x0, 0xf, {Routes.north}), + RTE({Routes.south, Routes.south_west}, 0x1, 0xf, {Routes.north_east}), + RTE({Routes.north}, 0x2, 0xe), + ] + + # We expect the result + # N NE -> 000X -> S SW + # ? -> 001X -> N + expected = [ + RTE({Routes.south, Routes.south_west}, 0x0, 0xe, + {Routes.north, Routes.north_east}), + RTE({Routes.north}, 0x2, 0xe), + ] + + # Try the minimisation + assert minimise(table, None) == expected + + +def test_minimise_fails_if_oversize(): + """Test minimisation of a simple routing table, we assume the C tests are + extensive! + """ + # Minimise the table: + # N -> 0000 -> S SW + # NE -> 0001 -> S SW + # ? -> 001X -> N + table = [ + RTE({Routes.south, Routes.south_west}, 0x0, 0xf, {Routes.north}), + RTE({Routes.south, Routes.south_west}, 0x1, 0xf, {Routes.north_east}), + RTE({Routes.north}, 0x2, 0xe), + ] + + # We expect the result to be: + # N NE -> 000X -> S SW + # ? -> 001X -> N + # Which is, clearly, more than 1 entry so an error should be raised + + with pytest.raises(MinimisationFailedError) as exc: + minimise(table, 1) + + assert exc.value.target_length == 1 + assert exc.value.final_length == 2 + assert exc.value.chip is None diff --git a/py_tests/test_ordered_covering.py b/py_tests/test_ordered_covering.py new file mode 100644 index 0000000..f4fca20 --- /dev/null +++ b/py_tests/test_ordered_covering.py @@ -0,0 +1,65 @@ +import pytest +from rig.routing_table import RoutingTableEntry as RTE +from rig.routing_table import Routes, MinimisationFailedError +from rig_routing_tables.ordered_covering import minimise + + +@pytest.mark.parametrize("target_length, no_raise", + ((None, False), (0, True))) +def test_ordered_covering_simple(target_length, no_raise): + """Test that a very simple routing table can be minimised, only one + merge should be made as there are no conflicts. + + Table:: + + 0000 -> N, NE + 0001 -> N, NE + 001X -> S + + Can be minimised to:: + + 000X -> N, NE + 001X -> S + """ + # Original table + table = [ + RTE({Routes.north, Routes.north_east}, 0b0000, 0b1111), + RTE({Routes.north, Routes.north_east}, 0b0001, 0b1111), + RTE({Routes.south}, 0b0010, 0b1110), + ] + + # Expected table + expected_table = [ + RTE({Routes.north, Routes.north_east}, 0b0000, 0b1110), + RTE({Routes.south}, 0b0010, 0b1110), + ] + + # Minimise and check the result + new_table = minimise(table, target_length, no_raise) + + assert new_table == expected_table + + +def test_ordered_covering_simple_fails_if_too_large(): + """Test that a very simple routing table can be minimised, and that an + exception is raised if that minimisation is still too large. + + Table:: + + 0000 -> N, NE + 0001 -> N, NE + 001X -> S + """ + # Original table + table = [ + RTE({Routes.north, Routes.north_east}, 0b0000, 0b1111), + RTE({Routes.north, Routes.north_east}, 0b0001, 0b1111), + RTE({Routes.south}, 0b0010, 0b1110), + ] + + with pytest.raises(MinimisationFailedError) as exc: + minimise(table, target_length=1) + + assert exc.value.target_length == 1 + assert exc.value.final_length == 2 + assert exc.value.chip is None diff --git a/py_tests/test_utils.py b/py_tests/test_utils.py new file mode 100644 index 0000000..0fc6aff --- /dev/null +++ b/py_tests/test_utils.py @@ -0,0 +1,44 @@ +from rig.routing_table import RoutingTableEntry as RTE +from rig.routing_table import Routes +from rig_routing_tables import utils + + +def test_table_to_cffi_and_back(): + """Test conversion of a Rig routing table to the CFFI format and conversion + back again. + """ + # Create the initial table + table = [RTE({r}, i, 0xffffffff) for i, r in enumerate(Routes)] + table += [ + RTE({Routes.north, Routes.south}, 0x80000000, 0xffffffff, + {Routes.east, Routes.west}), + ] + + # Convert the table to C format + c_table = utils.rig_to_c_table(table) + + # Check for equivalence + assert c_table.size == len(table) + for i, entry in enumerate(table): + c_entry = c_table.entries[i] + + assert c_entry.keymask.key == entry.key + assert c_entry.keymask.mask == entry.mask + + # Check the route is equivalent + route = 0x0 + for r in entry.route: + route |= 1 << r + + assert c_entry.route == route + + # Check the source is equivalent + source = 0x0 + for r in entry.sources: + source |= 1 << (r if r is not None else 31) + + assert c_entry.source == source + + # Convert back again + returned_table = utils.c_to_rig_table(c_table) + assert table == returned_table diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..8eb70b0 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,3 @@ +pytest>=2.6 +pytest-cov +flake8 diff --git a/rig_routing_tables/__init__.py b/rig_routing_tables/__init__.py new file mode 100644 index 0000000..86b5e1d --- /dev/null +++ b/rig_routing_tables/__init__.py @@ -0,0 +1,4 @@ +"""Faster C implementation of common routing table minimisers.""" +from rig_routing_tables.version import __version__ + +from rig_routing_tables import utils diff --git a/rig_routing_tables/cffi_compile.py b/rig_routing_tables/cffi_compile.py new file mode 100644 index 0000000..863cb13 --- /dev/null +++ b/rig_routing_tables/cffi_compile.py @@ -0,0 +1,55 @@ +import os +from cffi import FFI +ffi = FFI() + + +# Get the include directory +include_dir = os.path.join(os.path.dirname(__file__), "..", "include") + +ffi.set_source( + "_rig_routing_tables", + """ + #include "mtrie.h" + #include "routing_table.h" + #include "ordered_covering.h" + #include "cffi_utils.h" + """, + include_dirs=[include_dir, os.path.dirname(__file__)], + sources=[os.path.join(os.path.dirname(__file__), "cffi_utils.c")], +) + +ffi.cdef(""" + // Data structures + typedef struct _keymask_t + { + uint32_t key; // Key for the keymask + uint32_t mask; // Mask for the keymask + ...; + } keymask_t; + + typedef struct _entry_t + { + keymask_t keymask; // Key and mask + uint32_t route; // Routing direction + uint32_t source; // Source of packets arriving at this entry + ...; + } entry_t; + + typedef struct _table_t + { + unsigned int size; // Number of entries in the table + entry_t *entries; // Entries in the table + ...; + } table_t; + + // Table management + table_t* new_table(unsigned int size); + void free_table(table_t *table); + + // Minimisation methods + void mtrie_minimise(table_t *table); + void oc_minimise_na(table_t *table, unsigned int target_length); +""") + +if __name__ == "__main__": + ffi.compile() diff --git a/rig_routing_tables/cffi_utils.c b/rig_routing_tables/cffi_utils.c new file mode 100644 index 0000000..b236ae7 --- /dev/null +++ b/rig_routing_tables/cffi_utils.c @@ -0,0 +1,33 @@ +#include +#include +#include "cffi_utils.h" + +table_t *new_table(unsigned int size) +{ + // Malloc space for the table and associated entries + table_t *table = malloc(sizeof(table_t)); + table->size = 0; + table->entries = NULL; + + if (table != NULL) + { + table->entries = malloc(sizeof(entry_t) * size); + + if (table->entries != NULL) + { + table->size = size; + return table; + } + } + + return NULL; +} + +void free_table(table_t *table) +{ + // Free the entries and then the table + free(table->entries); + table->entries = NULL; + table->size = 0; + free(table); +} diff --git a/rig_routing_tables/cffi_utils.h b/rig_routing_tables/cffi_utils.h new file mode 100644 index 0000000..52171c7 --- /dev/null +++ b/rig_routing_tables/cffi_utils.h @@ -0,0 +1,5 @@ +#include "routing_table.h" + +// Table management methods (mostly here for the CFFI wrapper) +table_t *new_table(unsigned int size); +void free_table(table_t *table); diff --git a/rig_routing_tables/mtrie.py b/rig_routing_tables/mtrie.py new file mode 100644 index 0000000..f3d9427 --- /dev/null +++ b/rig_routing_tables/mtrie.py @@ -0,0 +1,42 @@ +from rig.routing_table import MinimisationFailedError +from rig_routing_tables.utils import rig_to_c_table, c_to_rig_table +from _rig_routing_tables import lib + + +def minimise(table, target_length): + """Minimise a Rig routing table using the m-Trie method of Ahmad and + Mahapatra [1]_. + + Parameters + ---------- + routing_table : [:py:class:`~rig.routing_table.RoutingTableEntry`, ...] + Routing table from which to remove entries which could be handled by + default routing. + target_length : int or None + Target length of the routing table. + + Raises + ------ + MinimisationFailedError + If the smallest table that can be produced is larger than + `target_length`. + + Returns + ------- + [:py:class:`~rig.routing_table.RoutingTableEntry`, ...] + Reduced routing table entries. + + [1]_ Ahmad, S. and Mahapatra, R.N., 2007. An efficient approach to on-chip + logic minimization. *Very Large Scale Integration (VLSI) Systems, IEEE + Transactions on*, 15(9), pp.1040-1050. + """ + # Convert the table to C format and minimise + c_table = rig_to_c_table(table) + lib.mtrie_minimise(c_table) + + # If the table is larger than the target length then raise an exception + if target_length is not None and c_table.size > target_length: + raise MinimisationFailedError(target_length, c_table.size) + + # Otherwise convert back to Rig form and return + return c_to_rig_table(c_table) diff --git a/rig_routing_tables/ordered_covering.py b/rig_routing_tables/ordered_covering.py new file mode 100644 index 0000000..8824ba7 --- /dev/null +++ b/rig_routing_tables/ordered_covering.py @@ -0,0 +1,43 @@ +from rig.routing_table import MinimisationFailedError +from rig_routing_tables.utils import rig_to_c_table, c_to_rig_table +from _rig_routing_tables import lib + + +def minimise(table, target_length, no_raise=False): + """Minimise a Rig routing table using the Ordered Covering algorithm [1]_. + + Parameters + ---------- + routing_table : [:py:class:`~rig.routing_table.RoutingTableEntry`, ...] + Routing table from which to remove entries which could be handled by + default routing. + target_length : int or None + Target length of the routing table. + + Raises + ------ + MinimisationFailedError + If the smallest table that can be produced is larger than + `target_length`. + + Returns + ------- + [:py:class:`~rig.routing_table.RoutingTableEntry`, ...] + Reduced routing table entries. + + [1]_ Mundy, A., Heathcote, J., Garside J.D., On-chip order-exploiting + routing table minimization for a multicast supercomputer network. In *High + Performance Switching and Routing (HPSR), 2016 International Conference on* + IEEE (In Press) + """ + # Convert the table to C format and minimise + c_table = rig_to_c_table(table) + lib.oc_minimise_na(c_table, target_length or 0) + + # If the table is larger than the target length then raise an exception + if (not no_raise and target_length is not None and + c_table.size > target_length): + raise MinimisationFailedError(target_length, c_table.size) + + # Otherwise convert back to Rig form and return + return c_to_rig_table(c_table) diff --git a/rig_routing_tables/utils.py b/rig_routing_tables/utils.py new file mode 100644 index 0000000..aadd4d5 --- /dev/null +++ b/rig_routing_tables/utils.py @@ -0,0 +1,77 @@ +"""Utilities for converting routing tables between different formats.""" +from rig.routing_table import RoutingTableEntry, Routes +from _rig_routing_tables import ffi, lib + + +def rig_to_c_table(table): + """Convert a Rig-format routing table into the format expected by the C + functions. + + Parameters + ---------- + table : [:py:class:`rig.routing_table.RoutingTableEntry`] + List of routing table entries. + + Returns + ------- + table + C format routing table. + """ + # Create the table + c_table_raw = lib.new_table(len(table)) + assert c_table_raw != ffi.NULL + c_table = ffi.gc(c_table_raw, lib.free_table) + + # Add the entries + for i, entry in enumerate(table): + # Get the relevant C entry + c_entry = c_table.entries[i] + + # Set up as required + c_entry.keymask.key = entry.key + c_entry.keymask.mask = entry.mask + + # Compute the route + c_entry.route = 0x0 + for r in entry.route: + c_entry.route |= 1 << r + + # Compute the source, including the MSB if None is present in the set + c_entry.source = 0x0 + for r in entry.sources: + c_entry.source |= 1 << (r if r is not None else 31) + + return c_table + + +def c_to_rig_table(c_table): + """Convert a C-format routing table into the format expected by Rig + methods. + + Parameters + ---------- + c_table : + C format routing table (e.g., as constructed by + :py:func:`rig_to_c_table`. + + Returns + ------- + table : [:py:class:`rig.routing_table.RoutingTableEntry`] + List of routing table entries. + """ + # Construct the table by iterating over the entries in the C format + table = list() + for entry in (c_table.entries[i] for i in range(c_table.size)): + # Get the source and route + route = {r for r in Routes if (1 << r) & entry.route} + source = {r for r in Routes if (1 << r) & entry.source} + + # Special bit indicating absence of knowledge about some sources + if entry.source >> 31: + source.add(None) + + # Add the new entry to the table + table.append(RoutingTableEntry(route, entry.keymask.key, + entry.keymask.mask, source)) + + return table diff --git a/rig_routing_tables/version.py b/rig_routing_tables/version.py new file mode 100644 index 0000000..6c8e6b9 --- /dev/null +++ b/rig_routing_tables/version.py @@ -0,0 +1 @@ +__version__ = "0.0.0" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..8f529b0 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[flake8] +exclude = rig_routing_tables/__init__.py diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..a927272 --- /dev/null +++ b/setup.py @@ -0,0 +1,50 @@ +from setuptools import setup, find_packages + +with open("rig_routing_tables/version.py", "r") as f: + exec(f.read()) + +setup( + name="rig_routing_tables", + version=__version__, + packages=find_packages(), + + # Files required by CFFI wrapper + package_data={ + 'rig_routing_tables': ['../include/*.h', + 'rig_routing_tables/cffi_utils.c', + 'rig_routing_tables/cffi_utils.h'] + }, + + # Metadata for PyPi + url="https://github.com/project-rig/rig_routing_tables", + author="The Rig Authors", + description="A C library (and CFFI Python Interface) for " + "routing table minimisation.", + license="GPLv2", + classifiers=[ + "Development Status :: 3 - Alpha", + + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + + "License :: OSI Approved :: GNU General Public License v2 (GPLv2)", + + "Operating System :: POSIX :: Linux", + "Operating System :: Microsoft :: Windows", + "Operating System :: MacOS", + + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + + "Topic :: Software Development :: Libraries", + ], + keywords="spinnaker cffi routing-table-minimization", + + # Build CFFI Interface + cffi_modules=["rig_routing_tables/cffi_compile.py:ffi"], + setup_requires=["cffi>=1.0.0", "rig>=1.0.0, <2.0.0"], + install_requires=["cffi>=1.0.0", "rig>=1.0.0, <2.0.0"], +) From f87e3b574d1ea325f82981b3ca27d824780419c8 Mon Sep 17 00:00:00 2001 From: Andrew Mundy Date: Mon, 6 Jun 2016 10:09:04 +0100 Subject: [PATCH 02/10] fixup! Merge fault --- {tests => c_tests}/test_remove_default_routes.c | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {tests => c_tests}/test_remove_default_routes.c (100%) diff --git a/tests/test_remove_default_routes.c b/c_tests/test_remove_default_routes.c similarity index 100% rename from tests/test_remove_default_routes.c rename to c_tests/test_remove_default_routes.c From 684adc915e987cba5575e15c66143bcd555b1529 Mon Sep 17 00:00:00 2001 From: Andrew Mundy Date: Wed, 8 Jun 2016 20:36:06 +0100 Subject: [PATCH 03/10] fixup! Add method for removing default routes --- py_tests/test_remove_default_routes.py | 39 +++++++++++++++++++++ rig_routing_tables/cffi_compile.py | 2 ++ rig_routing_tables/mtrie.py | 1 + rig_routing_tables/remove_default_routes.py | 38 ++++++++++++++++++++ 4 files changed, 80 insertions(+) create mode 100644 py_tests/test_remove_default_routes.py create mode 100644 rig_routing_tables/remove_default_routes.py diff --git a/py_tests/test_remove_default_routes.py b/py_tests/test_remove_default_routes.py new file mode 100644 index 0000000..b65f20d --- /dev/null +++ b/py_tests/test_remove_default_routes.py @@ -0,0 +1,39 @@ +import pytest + +from rig.routing_table import (Routes, RoutingTableEntry, + MinimisationFailedError) +from rig_routing_tables.remove_default_routes import minimise + + +def test_minimise_orthogonal_table(): + """Test for correct removal of default routes in an orthogonal table.""" + table = [ + RoutingTableEntry({Routes.north}, 0x0, 0xf, {Routes.south}), # Remove + RoutingTableEntry({Routes.north}, 0x1, 0xf, {Routes.north}), # Keep + RoutingTableEntry({Routes.north}, 0x2, 0xf, {None}), # Keep + RoutingTableEntry({Routes.north, Routes.south}, 0x3, 0xf, + {Routes.north, Routes.south}), # Keep + RoutingTableEntry({Routes.core(1)}, 0x4, 0xf, {Routes.core(1)}) # Keep + ] + assert minimise(table, len(table) - 1) == table[1:] + + +def test_minimise_nonorthogonal_table(): + """Test for correct removal of default routes in an orthogonal table.""" + table = [ + RoutingTableEntry({Routes.north}, 0x8, 0xf, {Routes.south}), # Remove + RoutingTableEntry({Routes.north}, 0x0, 0xf, {Routes.south}), # Keep + RoutingTableEntry({Routes.north}, 0x0, 0x8, {None}), # Keep + ] + assert minimise(table, None) == table[1:] + + +def test_minimise_oversized(): + table = [ + RoutingTableEntry({Routes.north}, i, 0xf, {None}) for i in range(10) + ] + + with pytest.raises(MinimisationFailedError) as exc: + minimise(table, 5) + assert "10" in str(exc.value) + assert "5" in str(exc.value) diff --git a/rig_routing_tables/cffi_compile.py b/rig_routing_tables/cffi_compile.py index 863cb13..c3c0ae8 100644 --- a/rig_routing_tables/cffi_compile.py +++ b/rig_routing_tables/cffi_compile.py @@ -10,6 +10,7 @@ "_rig_routing_tables", """ #include "mtrie.h" + #include "remove_default_routes.h" #include "routing_table.h" #include "ordered_covering.h" #include "cffi_utils.h" @@ -49,6 +50,7 @@ // Minimisation methods void mtrie_minimise(table_t *table); void oc_minimise_na(table_t *table, unsigned int target_length); + void remove_default_routes_minimise(table_t *table); """) if __name__ == "__main__": diff --git a/rig_routing_tables/mtrie.py b/rig_routing_tables/mtrie.py index f3d9427..eff9223 100644 --- a/rig_routing_tables/mtrie.py +++ b/rig_routing_tables/mtrie.py @@ -33,6 +33,7 @@ def minimise(table, target_length): # Convert the table to C format and minimise c_table = rig_to_c_table(table) lib.mtrie_minimise(c_table) + lib.remove_default_routes_minimise(c_table) # If the table is larger than the target length then raise an exception if target_length is not None and c_table.size > target_length: diff --git a/rig_routing_tables/remove_default_routes.py b/rig_routing_tables/remove_default_routes.py new file mode 100644 index 0000000..3bead95 --- /dev/null +++ b/rig_routing_tables/remove_default_routes.py @@ -0,0 +1,38 @@ +from rig.routing_table import MinimisationFailedError +from rig_routing_tables.utils import rig_to_c_table, c_to_rig_table +from _rig_routing_tables import lib + + +def minimise(table, target_length): + """Minimise a Rig routing table by removing all entries which could be + replaced by default routing. + + Parameters + ---------- + routing_table : [:py:class:`~rig.routing_table.RoutingTableEntry`, ...] + Routing table from which to remove entries which could be handled by + default routing. + target_length : int or None + Target length of the routing table. + + Raises + ------ + MinimisationFailedError + If the smallest table that can be produced is larger than + `target_length`. + + Returns + ------- + [:py:class:`~rig.routing_table.RoutingTableEntry`, ...] + Reduced routing table entries. + """ + # Convert the table to C format and minimise + c_table = rig_to_c_table(table) + lib.remove_default_routes_minimise(c_table) + + # If the table is larger than the target length then raise an exception + if target_length is not None and c_table.size > target_length: + raise MinimisationFailedError(target_length, c_table.size) + + # Otherwise convert back to Rig form and return + return c_to_rig_table(c_table) From 34efb751b2cee6cff8d29dc58f4c0fcae88791d9 Mon Sep 17 00:00:00 2001 From: Andrew Mundy Date: Wed, 8 Jun 2016 20:37:24 +0100 Subject: [PATCH 04/10] fixup! Correct docstrings --- rig_routing_tables/mtrie.py | 3 +-- rig_routing_tables/ordered_covering.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/rig_routing_tables/mtrie.py b/rig_routing_tables/mtrie.py index eff9223..d38e2d7 100644 --- a/rig_routing_tables/mtrie.py +++ b/rig_routing_tables/mtrie.py @@ -10,8 +10,7 @@ def minimise(table, target_length): Parameters ---------- routing_table : [:py:class:`~rig.routing_table.RoutingTableEntry`, ...] - Routing table from which to remove entries which could be handled by - default routing. + Routing table to minimise. target_length : int or None Target length of the routing table. diff --git a/rig_routing_tables/ordered_covering.py b/rig_routing_tables/ordered_covering.py index 8824ba7..635b4f7 100644 --- a/rig_routing_tables/ordered_covering.py +++ b/rig_routing_tables/ordered_covering.py @@ -9,8 +9,7 @@ def minimise(table, target_length, no_raise=False): Parameters ---------- routing_table : [:py:class:`~rig.routing_table.RoutingTableEntry`, ...] - Routing table from which to remove entries which could be handled by - default routing. + Routing table to minimise. target_length : int or None Target length of the routing table. From 544ee4a04b1df88fa658874857db19d26267296c Mon Sep 17 00:00:00 2001 From: Andrew Mundy Date: Wed, 8 Jun 2016 20:38:29 +0100 Subject: [PATCH 05/10] fixup! Setup doesn't require Rig --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a927272..670b40d 100644 --- a/setup.py +++ b/setup.py @@ -45,6 +45,6 @@ # Build CFFI Interface cffi_modules=["rig_routing_tables/cffi_compile.py:ffi"], - setup_requires=["cffi>=1.0.0", "rig>=1.0.0, <2.0.0"], + setup_requires=["cffi>=1.0.0"], install_requires=["cffi>=1.0.0", "rig>=1.0.0, <2.0.0"], ) From fc21dd7b1430681a2e737e1e9fbd5ab0646df2b3 Mon Sep 17 00:00:00 2001 From: Andrew Mundy Date: Wed, 8 Jun 2016 20:38:47 +0100 Subject: [PATCH 06/10] Call `exit' if a malloc fails Die hard rather than handling the error pleasantly. --- include/platform.h | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/include/platform.h b/include/platform.h index 604ea70..b3d60a5 100644 --- a/include/platform.h +++ b/include/platform.h @@ -27,7 +27,17 @@ #else #include - #define MALLOC malloc + static inline void * safe_malloc(uint bytes) + { + void* p = malloc(bytes); + if (p == NULL) + { + exit(-1); + } + return p; + } + + #define MALLOC safe_malloc #define FREE free #endif From 3267d0ce616fb16968054f289ffb594a12737b86 Mon Sep 17 00:00:00 2001 From: Andrew Mundy Date: Wed, 8 Jun 2016 21:30:41 +0100 Subject: [PATCH 07/10] fixup! Add tools for reading/writing routing tables to file --- py_tests/test_utils.py | 107 ++++++++++++++++++++++++++++++++++ rig_routing_tables/utils.py | 112 ++++++++++++++++++++++++++++++++++++ 2 files changed, 219 insertions(+) diff --git a/py_tests/test_utils.py b/py_tests/test_utils.py index 0fc6aff..476c0a7 100644 --- a/py_tests/test_utils.py +++ b/py_tests/test_utils.py @@ -1,6 +1,9 @@ +import pytest from rig.routing_table import RoutingTableEntry as RTE from rig.routing_table import Routes from rig_routing_tables import utils +import struct +import tempfile def test_table_to_cffi_and_back(): @@ -42,3 +45,107 @@ def test_table_to_cffi_and_back(): # Convert back again returned_table = utils.c_to_rig_table(c_table) assert table == returned_table + + +def test_read_routing_table_from_file(): + """Test reading a routing table from file.""" + # Write two routing tables to file in the appropriate format, then read + # back and ensure the result is correct. + fp = tempfile.TemporaryFile() + + # Write the first routing table: + # (1, 2): + # S -> 00000000000000000000000000000000 -> NE 3 + # E -> 00000000000000000000000000000001 -> S + fp.write(struct.pack("<2BH", 1, 2, 2)) + fp.write(struct.pack("<4I", 0x0, 0xffffffff, 0b100000, 0b1000000010)) + fp.write(struct.pack("<4I", 0x1, 0xffffffff, 0b000001, 0b0000100000)) + + # Write the second routing table: + # (0, 0): + # ? -> 00000000000000000000000000000000 -> 1 + fp.write(struct.pack("<2BH", 0, 0, 1)) + fp.write(struct.pack("<4I", 0x0, 0xffffffff, 1 << 31, 0b10000000)) + + # Check that the table is expected + fp.seek(0) + assert dict(utils.read_routing_tables(fp)) == { + (1, 2): [ + RTE({Routes.north_east, Routes.core(3)}, 0x0, 0xffffffff, + {Routes.south}), + RTE({Routes.south}, 0x1, 0xffffffff, {Routes.east}), + ], + (0, 0): [ + RTE({Routes.core(1)}, 0x0, 0xffffffff), + ], + } + + fp.close() + + +def test_read_routing_table_from_file_truncated_header(): + fp = tempfile.TemporaryFile() + + # Write the first routing table: + # (1, 2): + # S -> 00000000000000000000000000000000 -> NE 3 + # E -> 00000000000000000000000000000001 -> S + fp.write(struct.pack("<2BH", 1, 2, 2)) + fp.write(struct.pack("<4I", 0x0, 0xffffffff, 0b100000, 0b1000000010)) + fp.write(struct.pack("<4I", 0x1, 0xffffffff, 0b000001, 0b0000100000)) + + # Write a truncated header + fp.write(struct.pack("<2B", 0, 0)) + + # Check that an exception is raised + fp.seek(0) + with pytest.raises(AssertionError): + dict(utils.read_routing_tables(fp)) + + fp.close() + + +def test_read_routing_table_from_file_truncated_entry(): + fp = tempfile.TemporaryFile() + + # Write the first routing table: + # (1, 2): + # S -> 00000000000000000000000000000000 -> NE 3 + # E -> 00000000000000000000000000000001 -> S + fp.write(struct.pack("<2BH", 1, 2, 2)) + fp.write(struct.pack("<4I", 0x0, 0xffffffff, 0b100000, 0b1000000010)) + fp.write(struct.pack("<4I", 0x1, 0xffffffff, 0b000001, 0b0000100000)) + + # Write the second routing table (truncated) + # (0, 0): + # ? -> 00000000000000000000000000000000 -> 1 + fp.write(struct.pack("<2BHI", 0, 0, 1, 0)) + + # Check that an exception is raised + fp.seek(0) + with pytest.raises(AssertionError): + dict(utils.read_routing_tables(fp)) + + fp.close() + + +def test_write_routing_tables(): + """Write to file and read back to check the table is correct.""" + tables = { + (0, 0): [ + RTE({Routes.south, Routes.south_west}, 0x0, 0xf, {Routes.north}), + RTE({Routes.south, Routes.south}, 0x1, 0xf, {Routes.north_east}), + RTE({Routes.north}, 0x2, 0xe), + ], + (0, 1): [ + RTE({Routes.north}, 0x2, 0xe), + ] + } + + # Check on writing and reading + fp = tempfile.TemporaryFile() + utils.write_routing_tables(tables, fp) + fp.seek(0) + assert dict(utils.read_routing_tables(fp)) == tables + + fp.close() diff --git a/rig_routing_tables/utils.py b/rig_routing_tables/utils.py index aadd4d5..eaccf0b 100644 --- a/rig_routing_tables/utils.py +++ b/rig_routing_tables/utils.py @@ -1,6 +1,8 @@ """Utilities for converting routing tables between different formats.""" from rig.routing_table import RoutingTableEntry, Routes from _rig_routing_tables import ffi, lib +from six import iteritems +import struct def rig_to_c_table(table): @@ -75,3 +77,113 @@ def c_to_rig_table(c_table): entry.keymask.mask, source)) return table + + +def read_routing_tables(fp): + """Read routing tables from a file. + + Parameters + ---------- + fp : file object + File object containing routing tables. + + Each routing table starts with two bytes representing the x and y + co-ordinates of the chip to which the routing table relates (a byte for + each of the co-ordinates) and a short (two bytes) representing the number + of entries in the following table. A table entry consists of four words, + one each for the key, the mask, the source and the route fields. + + Yields + ------ + (x, y) + Co-ordinates of the routing table. + [:py:class:`rig.routing_table.RoutingTableEntry`, ...] + Routing table found at this co-ordinate. + """ + for xy, c_table in read_routing_tables_in_c_format(fp): + yield xy, c_to_rig_table(c_table) + + +def write_routing_tables(tables, fp): + """Write routing tables to a file. + + Parameters + ---------- + tables : {(x, y): [:py:class:`rig.routing_table.RoutingTableEntry`, ...]} + Dictionary mapping chip co-ordinates to the routing table associated + with that chip. + fp : file object + """ + write_routing_tables_from_c_format( + {chip: rig_to_c_table(table) for chip, table in iteritems(tables)}, fp + ) + + +def read_routing_tables_in_c_format(fp): + """Read routing tables from a file. + + Parameters + ---------- + fp : file object + File object containing routing tables. + + Yields + ------ + (x, y) + Co-ordinates of the routing table. + c_table + Routing table found at this co-ordinate (in C format). + """ + while True: + # Try to read the header + header = fp.read(4) + if header == b'': + break + assert len(header) == 4, "Truncated header" + + x, y, length = struct.unpack("<2BH", header) + + # Create the routing table + c_table_raw = lib.new_table(length) + assert c_table_raw != ffi.NULL + c_table = ffi.gc(c_table_raw, lib.free_table) + + # Read in the entries + for entry in (c_table.entries[i] for i in range(length)): + data = fp.read(16) + assert len(data) == 16, "Truncated entry" + (entry.keymask.key, entry.keymask.mask, + entry.source, entry.route) = struct.unpack("<4I", data) + + yield (x, y), c_table + + +def write_routing_tables_from_c_format(tables, fp): + """Write routing tables to a file. + + Parameters + ---------- + tables : {(x, y): c_table} + Dictionary mapping chip co-ordinates to the routing table associated + with that chip. + fp : file object + """ + # Write each table in turn, buffering in memory to save writes + for (x, y), table in iteritems(tables): + # Create the buffer + data = bytearray(4 + 16*table.size) + + # Write the header + struct.pack_into("<2BH", data, 0, x, y, table.size) + + # Dump in the entries + for i in range(table.size): + entry = table.entries[i] + struct.pack_into("<4I", data, 4 + 16*i, + entry.keymask.key, + entry.keymask.mask, + entry.source, + entry.route) + + # Write out + fp.write(data) From aa7b1eef812bf560692ad959af0990baa9d1db5a Mon Sep 17 00:00:00 2001 From: Andrew Mundy Date: Mon, 13 Jun 2016 16:29:14 +0900 Subject: [PATCH 08/10] fixup! Ordered Covering should first sort table entries --- py_tests/test_ordered_covering.py | 22 ++++++++++++++++++++++ rig_routing_tables/ordered_covering.py | 8 ++++++++ 2 files changed, 30 insertions(+) diff --git a/py_tests/test_ordered_covering.py b/py_tests/test_ordered_covering.py index f4fca20..03a46a2 100644 --- a/py_tests/test_ordered_covering.py +++ b/py_tests/test_ordered_covering.py @@ -63,3 +63,25 @@ def test_ordered_covering_simple_fails_if_too_large(): assert exc.value.target_length == 1 assert exc.value.final_length == 2 assert exc.value.chip is None + + +def test_sort_table_before_minimisation(): + """Test that the routing table is reordered before insertion. + + The following table won't minimise, but should be reordered: + + 01XX -> N + 001X -> NE + 0000 -> E + """ + table = [ + RTE({Routes.north}, 0x4, 0xc), + RTE({Routes.north_east}, 0x2, 0xe), + RTE({Routes.east}, 0x0, 0xf), + ] + + # Minimise (should re-order) + minimised = minimise(list(table), None) + + # Assert the reordering occurred + assert minimised == table[-1::-1] diff --git a/rig_routing_tables/ordered_covering.py b/rig_routing_tables/ordered_covering.py index 635b4f7..c155323 100644 --- a/rig_routing_tables/ordered_covering.py +++ b/rig_routing_tables/ordered_covering.py @@ -3,6 +3,11 @@ from _rig_routing_tables import lib +def get_generality(entry): + """Return the generality (number of Xs) of the entry.""" + return sum((~(entry.key | entry.mask) >> i) & 1 for i in range(31)) + + def minimise(table, target_length, no_raise=False): """Minimise a Rig routing table using the Ordered Covering algorithm [1]_. @@ -29,6 +34,9 @@ def minimise(table, target_length, no_raise=False): Performance Switching and Routing (HPSR), 2016 International Conference on* IEEE (In Press) """ + # Initially sort the table in ascending order of generality + table.sort(key=get_generality) + # Convert the table to C format and minimise c_table = rig_to_c_table(table) lib.oc_minimise_na(c_table, target_length or 0) From 8f5dddc9b70b175586e193f97e5a95990331f58f Mon Sep 17 00:00:00 2001 From: Andrew Mundy Date: Mon, 13 Jun 2016 16:29:55 +0900 Subject: [PATCH 09/10] fixup! Definitely no Windows support at the moment --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 670b40d..50edf3c 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,6 @@ "License :: OSI Approved :: GNU General Public License v2 (GPLv2)", "Operating System :: POSIX :: Linux", - "Operating System :: Microsoft :: Windows", "Operating System :: MacOS", "Programming Language :: Python :: 2", From a01c19155f59ed7735a1632849177b2d29384705 Mon Sep 17 00:00:00 2001 From: Andrew Mundy Date: Mon, 13 Jun 2016 16:43:09 +0900 Subject: [PATCH 10/10] fixup! Flake8 --- py_tests/test_ordered_covering.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py_tests/test_ordered_covering.py b/py_tests/test_ordered_covering.py index 03a46a2..1ea4bf3 100644 --- a/py_tests/test_ordered_covering.py +++ b/py_tests/test_ordered_covering.py @@ -67,7 +67,7 @@ def test_ordered_covering_simple_fails_if_too_large(): def test_sort_table_before_minimisation(): """Test that the routing table is reordered before insertion. - + The following table won't minimise, but should be reordered: 01XX -> N