Merge pull request #1088 from pq-code-package/slothy_for_neoverse_n1

SLOTHY: Optimize AArch64 arithmetic backend for Neoverse-N1

Author: hanno-becker

.github/workflows/slothy.yml

@@ -17,7 +17,7 @@ jobs:
     uses: ./.github/workflows/ci_ec2_reusable.yml
     with:
       name: SLOTHY
-      ec2_instance_type: c8g.4xlarge
+      ec2_instance_type: c8g.8xlarge
       ec2_ami: ubuntu-latest (custom AMI)
       ec2_ami_id: ami-08ddb0acd99dc3d33 # aarch64, ubuntu-latest, 64g
       lint: false

dev/aarch64_clean/src/intt.S

@@ -235,27 +235,6 @@ MLK_ASM_FN_SYMBOL(intt_asm)
         mov inp, in
         mov count, #8
 
-intt_scale_start:
-
-        ldr q_data0, [inp, #(16*0)]
-        ldr q_data1, [inp, #(16*1)]
-        ldr q_data2, [inp, #(16*2)]
-        ldr q_data3, [inp, #(16*3)]
-
-        mul_ninv data0, data1, data2, data3, data0, data1, data2, data3
-        // Bounds: Absolute value < q
-
-        str q_data0, [inp], #64
-        str q_data1, [inp, #(-64 + 16*1)]
-        str q_data2, [inp, #(-64 + 16*2)]
-        str q_data3, [inp, #(-64 + 16*3)]
-
-        subs count, count, #1
-        cbnz count, intt_scale_start
-
-        mov inp, in
-        mov count, #8
-
         .p2align 2
 intt_layer4567_start:
 
@@ -266,6 +245,9 @@ intt_layer4567_start:
 
         transpose4 data // manual ld4
 
+        mul_ninv data0, data1, data2, data3, data0, data1, data2, data3
+        // Bounds: Absolute value < q
+
         load_next_roots_67
 
         // Layer 7

dev/aarch64_opt/src/Makefile

@@ -10,7 +10,7 @@
 TARGET_ISA=Arm_AArch64
 
 # MicroArch target to optimize for
-TARGET_MICROARCH=Arm_Cortex_A55
+TARGET_MICROARCH=Arm_Neoverse_N1_experimental
 
 SLOTHY_EXTRA_FLAGS ?=
 
@@ -51,14 +51,15 @@ ntt.S: ../../aarch64_clean/src/ntt.S $(COMMON_H)
 
 intt.S: ../../aarch64_clean/src/intt.S $(COMMON_H)
 	slothy-cli $(TARGET_ISA) $(TARGET_MICROARCH) $< -o $@ -l intt_layer123_start -l intt_layer4567_start $(SLOTHY_FLAGS) $(RESERVE_X_ONLY_FLAG)
+
 polyvec_basemul_acc_montgomery_cached_asm_k2.S: ../../aarch64_clean/src/polyvec_basemul_acc_montgomery_cached_asm_k2.S $(COMMON_H)
-	slothy-cli $(TARGET_ISA) $(TARGET_MICROARCH) $< -o $@ -l polyvec_basemul_acc_montgomery_cached_k2_loop_start $(SLOTHY_FLAGS) $(RESERVE_X_ONLY_FLAG)
+	slothy-cli $(TARGET_ISA) $(TARGET_MICROARCH) $< -o $@ -l polyvec_basemul_acc_montgomery_cached_k2_loop_start -ctimeout=60 $(SLOTHY_FLAGS) $(RESERVE_X_ONLY_FLAG)
 
 polyvec_basemul_acc_montgomery_cached_asm_k3.S: ../../aarch64_clean/src/polyvec_basemul_acc_montgomery_cached_asm_k3.S $(COMMON_H)
-	slothy-cli $(TARGET_ISA) $(TARGET_MICROARCH) $< -o $@ -l polyvec_basemul_acc_montgomery_cached_k3_loop_start $(SLOTHY_FLAGS) $(RESERVE_X_ONLY_FLAG)
+	slothy-cli $(TARGET_ISA) $(TARGET_MICROARCH) $< -o $@ -l polyvec_basemul_acc_montgomery_cached_k3_loop_start -ctimeout=60 $(SLOTHY_FLAGS) $(RESERVE_X_ONLY_FLAG)
 
 polyvec_basemul_acc_montgomery_cached_asm_k4.S: ../../aarch64_clean/src/polyvec_basemul_acc_montgomery_cached_asm_k4.S $(COMMON_H)
-	slothy-cli $(TARGET_ISA) $(TARGET_MICROARCH) $< -o $@ -l polyvec_basemul_acc_montgomery_cached_k4_loop_start $(SLOTHY_FLAGS) $(RESERVE_X_ONLY_FLAG)
+	slothy-cli $(TARGET_ISA) $(TARGET_MICROARCH) $< -o $@ -l polyvec_basemul_acc_montgomery_cached_k4_loop_start -ctimeout=60 $(SLOTHY_FLAGS) $(RESERVE_X_ONLY_FLAG)
 
 
 
@@ -71,10 +72,15 @@ poly_reduce_asm.S: ../../aarch64_clean/src/poly_reduce_asm.S $(COMMON_H)
 	slothy-cli $(TARGET_ISA) $(TARGET_MICROARCH) $< -o $@ -l poly_reduce_loop_start $(SLOTHY_FLAGS) $(RESERVE_ALL_FLAG)
 
 poly_mulcache_compute_asm.S: ../../aarch64_clean/src/poly_mulcache_compute_asm.S $(COMMON_H)
-	slothy-cli $(TARGET_ISA) $(TARGET_MICROARCH) $< -o $@ -l poly_mulcache_compute_loop_start $(SLOTHY_FLAGS) $(RESERVE_ALL_FLAG)
+        # Unroll twice since kernel is small
+	slothy-cli $(TARGET_ISA) $(TARGET_MICROARCH) $< -o $@ -l poly_mulcache_compute_loop_start $(SLOTHY_FLAGS) $(RESERVE_ALL_FLAG) -c sw_pipelining.unroll=2
 
 poly_tobytes_asm.S: ../../aarch64_clean/src/poly_tobytes_asm.S $(COMMON_H)
-	slothy-cli $(TARGET_ISA) $(TARGET_MICROARCH) $< -o $@ -l poly_tobytes_loop_start $(SLOTHY_FLAGS) $(RESERVE_ALL_FLAG) -c sw_pipelining.unroll=4
+        # Somehow SLOTHY tends to mark almost all instructions as 'early', which isn't
+        # wrong but looks a bit odd in the optimized code. Minimize overlapping to avoid this.
+        #
+        # Unroll twice since kernel is small
+	slothy-cli $(TARGET_ISA) $(TARGET_MICROARCH) $< -o $@ -l poly_tobytes_loop_start $(SLOTHY_FLAGS) $(RESERVE_ALL_FLAG) -c sw_pipelining.unroll=2 -c sw_pipelining.minimize_overlapping -ctimeout=60
 
 # At the moment, SLOTHY can't process rej_uniform_asm.S
 rej_uniform_asm.S: ../../aarch64_clean/src/rej_uniform_asm.S $(COMMON_H)