Merge pull request ethereum#1521 from hwwhww/rework_helper2

Rework helper functions - part 1

Author: hwwhww

eth/beacon/constants.py

@@ -5,4 +5,8 @@
 # The size of 3 bytes in integer
 # sample_range = 2 ** (3 * 8) = 2 ** 24 = 16777216
 # sample_range = 16777216
-SAMPLE_RANGE = 16777216
+
+# Entropy is consumed from the seed in 3-byte (24 bit) chunks.
+RAND_BYTES = 3
+# The highest possible result of the RNG.
+RAND_MAX = 2 ** (RAND_BYTES * 8) - 1

eth/beacon/enums/validator_status_codes.py

@@ -5,6 +5,5 @@ class ValidatorStatusCode(IntEnum):
     PENDING_ACTIVATION = 0
     ACTIVE = 1
     PENDING_EXIT = 2
-    PENDING_WITHDRAW = 3
-    WITHDRAWN = 4
-    PENALIZED = 127
+    EXITED_WITHOUT_PENALTY = 3
+    EXITED_WITH_PENALTY = 4

eth/beacon/helpers.py

@@ -23,6 +23,9 @@
 )
 
 from eth.beacon.block_committees_info import BlockCommitteesInfo
+from eth.beacon.enums.validator_status_codes import (
+    ValidatorStatusCode,
+)
 from eth.beacon.types.shard_and_committees import (
     ShardAndCommittee,
 )
@@ -37,6 +40,7 @@
     from eth.beacon.types.attestation_records import AttestationRecord  # noqa: F401
     from eth.beacon.types.blocks import BaseBeaconBlock  # noqa: F401
     from eth.beacon.types.crystallized_states import CrystallizedState  # noqa: F401
+    from eth.beacon.types.states import BeaconState  # noqa: F401
     from eth.beacon.types.validator_records import ValidatorRecord  # noqa: F401
 
 
@@ -170,6 +174,9 @@ def get_shards_and_committees_for_slot(
         crystallized_state: 'CrystallizedState',
         slot: int,
         cycle_length: int) -> Iterable[ShardAndCommittee]:
+    """
+    FIXME
+    """
     if len(crystallized_state.shard_and_committee_for_slots) != cycle_length * 2:
         raise ValueError(
             "Length of shard_and_committee_for_slots != cycle_length * 2"
@@ -192,6 +199,7 @@ def get_attestation_indices(crystallized_state: 'CrystallizedState',
                             attestation: 'AttestationRecord',
                             cycle_length: int) -> Iterable[int]:
     """
+    FIXME
     Return committee of the given attestation.
     """
     shard_id = attestation.shard_id
@@ -207,64 +215,30 @@ def get_attestation_indices(crystallized_state: 'CrystallizedState',
             yield from shard_and_committee.committee
 
 
-@to_tuple
-def get_active_validator_indices(dynasty: int,
-                                 validators: Iterable['ValidatorRecord']) -> Iterable[int]:
+def get_active_validator_indices(validators: Sequence['ValidatorRecord']) -> Tuple[int, ...]:
     """
-    TODO: Logic changed in the latest spec, will have to update when we add validator
-    rotation logic.
-    https://github.com/ethereum/eth2.0-specs/commit/52cf7f943dc99cfd27db9fb2c03c692858e2a789#diff-a08ecec277db4a6ed0b3635cfadc9af1  # noqa: E501
+    Gets indices of active validators from ``validators``.
     """
-    o = []
-    for index, validator in enumerate(validators):
-        if (validator.start_dynasty <= dynasty and dynasty < validator.end_dynasty):
-            o.append(index)
-    return o
+    return tuple(
+        i for i, v in enumerate(validators)
+        if v.status in [ValidatorStatusCode.ACTIVE, ValidatorStatusCode.PENDING_EXIT]
+    )
 
 
 #
 # Shuffling
 #
-def _get_shuffling_committee_slot_portions(
-        active_validators_size: int,
-        cycle_length: int,
-        min_committee_size: int,
-        shard_count: int) -> Tuple[int, int]:
-    """
-    Return committees number per slot and slots number per committee.
-    """
-    # If there are enough active validators to form committees for every slot
-    if active_validators_size >= cycle_length * min_committee_size:
-        # One slot can be attested by many committees, but not more than shard_count // cycle_length
-        committees_per_slot = min(
-            active_validators_size // cycle_length // (min_committee_size * 2) + 1,
-            shard_count // cycle_length
-        )
-        # One committee only has to attest one slot
-        slots_per_committee = 1
-    else:
-        # One slot can only be asttested by one committee
-        committees_per_slot = 1
-        # One committee has to asttest more than one slot
-        slots_per_committee = 1
-        bound = cycle_length * min(min_committee_size, active_validators_size)
-        while(active_validators_size * slots_per_committee < bound):
-            slots_per_committee *= 2
-
-    return committees_per_slot, slots_per_committee
-
-
 @to_tuple
 def _get_shards_and_committees_for_shard_indices(
         shard_indices: Sequence[Sequence[int]],
-        shard_start: int,
+        start_shard: int,
         shard_count: int) -> Iterable[ShardAndCommittee]:
     """
     Returns filled [ShardAndCommittee] tuple.
     """
     for index, indices in enumerate(shard_indices):
         yield ShardAndCommittee(
-            shard=(shard_start + index) % shard_count,
+            shard=(start_shard + index) % shard_count,
             committee=indices
         )
 
@@ -273,14 +247,13 @@ def _get_shards_and_committees_for_shard_indices(
 def get_new_shuffling(*,
                       seed: Hash32,
                       validators: Sequence['ValidatorRecord'],
-                      dynasty: int,
                       crosslinking_start_shard: int,
                       cycle_length: int,
-                      min_committee_size: int,
+                      target_committee_size: int,
                       shard_count: int) -> Iterable[Iterable[ShardAndCommittee]]:
     """
     Return shuffled ``shard_and_committee_for_slots`` (``[[ShardAndCommittee]]``) of
-    the given active ``validators``.
+    the given active ``validators`` using ``seed`` as entropy.
 
     Two-dimensional:
     The first layer is ``slot`` number
@@ -319,27 +292,26 @@ def get_new_shuffling(*,
                     ShardAndCommittee(shard_id=5, committee=[7, 3, 11]),
                 ],
             ]
-
-    NOTE: The spec might be updated to output an array rather than an array of arrays.
     """
-    active_validators = get_active_validator_indices(dynasty, validators)
+    active_validators = get_active_validator_indices(validators)
     active_validators_size = len(active_validators)
     committees_per_slot = clamp(
         1,
         shard_count // cycle_length,
-        active_validators_size // cycle_length // (min_committee_size * 2) + 1,
+        active_validators_size // cycle_length // target_committee_size,
     )
+    # Shuffle with seed
     shuffled_active_validator_indices = shuffle(active_validators, seed)
 
-    # Split the shuffled list into cycle_length pieces
+    # Split the shuffled list into epoch_length pieces
     validators_per_slot = split(shuffled_active_validator_indices, cycle_length)
     for index, slot_indices in enumerate(validators_per_slot):
         # Split the shuffled list into committees_per_slot pieces
         shard_indices = split(slot_indices, committees_per_slot)
-        shard_id_start = crosslinking_start_shard + index * committees_per_slot
+        start_shard = crosslinking_start_shard + index * committees_per_slot
         yield _get_shards_and_committees_for_shard_indices(
             shard_indices,
-            shard_id_start,
+            start_shard,
             shard_count,
         )
 
@@ -356,6 +328,7 @@ def get_block_committees_info(parent_block: 'BaseBeaconBlock',
         cycle_length,
     )
     """
+    FIXME
     Return the block committees and proposer info with BlockCommitteesInfo pack.
     """
     # `proposer_index_in_committee` th attester in `shard_and_committee`

eth/beacon/utils/random.py

@@ -3,6 +3,7 @@
     Any,
     Iterable,
     Sequence,
+    Tuple,
     TypeVar,
 )
 
@@ -17,70 +18,81 @@
     blake,
 )
 from eth.beacon.constants import (
-    SAMPLE_RANGE,
+    RAND_BYTES,
+    RAND_MAX,
 )
 
 
 TItem = TypeVar('TItem')
 
 
+@to_tuple
 def shuffle(values: Sequence[Any],
             seed: Hash32) -> Iterable[Any]:
     """
     Returns the shuffled ``values`` with seed as entropy.
     Mainly for shuffling active validators in-protocol.
 
-    Spec: https://github.com/ethereum/eth2.0-specs/blob/0941d592de7546a428066c0473fd1000a7e3e3af/specs/beacon-chain.md#helper-functions  # noqa: E501
+    Spec: https://github.com/ethereum/eth2.0-specs/blob/70cef14a08de70e7bd0455d75cf380eb69694bfb/specs/core/0_beacon-chain.md#helper-functions  # noqa: E501
     """
     values_count = len(values)
 
-    if values_count > SAMPLE_RANGE:
+    # The range of the RNG places an upper-bound on the size of the list that
+    # may be shuffled. It is a logic error to supply an oversized list.
+    if values_count >= RAND_MAX:
         raise ValueError(
-            "values_count (%s) should less than or equal to SAMPLE_RANGE (%s)." %
-            (values_count, SAMPLE_RANGE)
+            "values_count (%s) should less than RAND_MAX (%s)." %
+            (values_count, RAND_MAX)
         )
 
     output = [x for x in values]
     source = seed
     index = 0
-    while index < values_count:
-        # Re-hash the source
+    while index < values_count - 1:
+        # Re-hash the `source` to obtain a new pattern of bytes.
         source = blake(source)
-        for position in range(0, 30, 3):  # gets indices 3 bytes at a time
-            # Select a 3-byte sampled int
-            sample_from_source = int.from_bytes(source[position:position + 3], 'big')
-            # `remaining` is the size of remaining indices of this round
+
+        # Iterate through the `source` bytes in 3-byte chunks.
+        for position in range(0, 32 - (32 % RAND_BYTES), RAND_BYTES):
+            # Determine the number of indices remaining in `values` and exit
+            # once the last index is reached.
             remaining = values_count - index
-            if remaining == 0:
+            if remaining == 1:
                 break
 
-            # Set a random maximum bound of sample_from_source
-            rand_max = SAMPLE_RANGE - SAMPLE_RANGE % remaining
+            # Read 3-bytes of `source` as a 24-bit big-endian integer.
+            sample_from_source = int.from_bytes(
+                source[position:position + RAND_BYTES], 'big'
+            )
 
-            # Select `replacement_position` with the given `sample_from_source` and `remaining`
-            if sample_from_source < rand_max:
-                # Use random number to get `replacement_position`, where it's not `index`
+            # Sample values greater than or equal to `sample_max` will cause
+            # modulo bias when mapped into the `remaining` range.
+            sample_max = RAND_MAX - RAND_MAX % remaining
+
+            # Perform a swap if the consumed entropy will not cause modulo bias.
+            if sample_from_source < sample_max:
+                # Select a replacement index for the current index.
                 replacement_position = (sample_from_source % remaining) + index
-                # Swap the index-th and replacement_position-th elements
+                # Swap the current index with the replacement index.
                 (output[index], output[replacement_position]) = (
                     output[replacement_position],
                     output[index]
                 )
                 index += 1
             else:
+                # The sample causes modulo bias. A new sample should be read.
                 pass
 
     return output
 
 
-@to_tuple
-def split(seq: Sequence[TItem], pieces: int) -> Iterable[Any]:
+def split(values: Sequence[TItem], split_count: int) -> Tuple[Any, ...]:
     """
-    Returns the split ``seq`` in ``pieces`` pieces in protocol.
-    Spec: https://github.com/ethereum/eth2.0-specs/blob/0941d592de7546a428066c0473fd1000a7e3e3af/specs/beacon-chain.md#helper-functions  # noqa: E501
+    Returns the split ``values`` in ``split_count`` pieces in protocol.
+    Spec: https://github.com/ethereum/eth2.0-specs/blob/70cef14a08de70e7bd0455d75cf380eb69694bfb/specs/core/0_beacon-chain.md#helper-functions  # noqa: E501
     """
-    list_length = len(seq)
-    return [
-        seq[(list_length * i // pieces): (list_length * (i + 1) // pieces)]
-        for i in range(pieces)
-    ]
+    list_length = len(values)
+    return tuple(
+        values[(list_length * i // split_count): (list_length * (i + 1) // split_count)]
+        for i in range(split_count)
+    )

eth/utils/numeric.py

@@ -99,3 +99,29 @@ def clamp(inclusive_lower_bound: int,
         return inclusive_upper_bound
     else:
         return value
+
+
+def integer_squareroot(value: int) -> int:
+    """
+    Return the largest integer ``x`` such that ``x**2 <= value``.
+    Ref: https://en.wikipedia.org/wiki/Integer_square_root
+    """
+    if not isinstance(value, int) or isinstance(value, bool):
+        raise ValueError(
+            "Value must be an integer: Got: {0}".format(
+                type(value),
+            )
+        )
+    if value < 0:
+        raise ValueError(
+            "Value cannot be negative: Got: {0}".format(
+                value,
+            )
+        )
+
+    x = value
+    y = (x + 1) // 2
+    while y < x:
+        x = y
+        y = (x + value // x) // 2
+    return x

tests/beacon/conftest.py

@@ -1,9 +1,20 @@
 import pytest
 
+from eth_utils import denoms
+
+from eth.constants import (
+    ZERO_HASH32,
+)
 import eth.utils.bls as bls
 from eth.utils.blake import blake
 
+from eth.beacon.enums.validator_status_codes import (
+    ValidatorStatusCode,
+)
 from eth.beacon.state_machines.forks.serenity.configs import SERENITY_CONFIG
+from eth.beacon.types.validator_records import (
+    ValidatorRecord,
+)
 
 from eth.beacon.types.attestation_signed_data import (
     AttestationSignedData,
@@ -349,3 +360,24 @@ def max_attestation_count():
 @pytest.fixture
 def initial_fork_version():
     return SERENITY_CONFIG.INITIAL_FORK_VERSION
+
+
+#
+# genesis
+#
+@pytest.fixture
+def genesis_validators(init_validator_keys,
+                       init_randao,
+                       deposit_size):
+    return tuple(
+        ValidatorRecord(
+            pubkey=pub,
+            withdrawal_credentials=ZERO_HASH32,
+            randao_commitment=init_randao,
+            randao_skips=0,
+            balance=deposit_size * denoms.gwei,
+            status=ValidatorStatusCode.ACTIVE,
+            last_status_change_slot=0,
+            exit_seq=0,
+        ) for pub in init_validator_keys
+    )