Skip to content

Commit 60f3e26

Browse files
dkupnickidkupnicki
authored
Merge pull request #5 from AmpereComputingAI/daniel/ampereone
add benchmark results
2 parents c0789ab + 779a37b commit 60f3e26

File tree

5 files changed

+193
-0
lines changed

5 files changed

+193
-0
lines changed

‎benchmarks/README.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,3 +37,15 @@ bash setup_deb.sh # works on Debian-based systems
3737
bash download_models.sh # uncomment preferred models in the file, by default llama3 q8_0 will be downloaded
3838
bash run.sh # modify to adjust number of threads available and other parameters
3939
```
40+
41+
## Speedup Ampere optimized llama.cpp (r1.2.6) vs llama.cpp (b3615)
42+
43+
| | Q4_K_M | Q4_K_4 | Speedup |
44+
|-----------------------|---|---|---|
45+
|token generation (t/s) | 191.76 | 289.99 | 1.51x |
46+
|prompt processing (t/s)| 265.64 | 542.87 | 2.04x |
47+
48+
| | Q8_0 | Q8R16 | Speedup |
49+
|-----------------------|---|---|---|
50+
|token generation (t/s) | 248.17 | 313.78 | 1.26x |
51+
|prompt processing (t/s)| 410.90 | 950.64 | 2.31x |

‎benchmarks/example_results/ampereone/ampere_optimized_llama.cpp_r1.2.6/Meta-Llama-3-8B-Instruct.Q4_K_4.gguf@PP128@TG256.csv‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
n_proc,n_threads,batch_size,prompt_size,output_tokens,pp_throughput_tps,pp_avg_latency_sec,tg_throughput_tps,tg_avg_latency_sec,pp+tg_throughput_tps,concurrency
2+
12,16,1,128,256,494.666894531456,3.10549999999999,39.6294901780034,0.3028095703125,56.7131481458689,12
3+
8,24,1,128,256,469.315399811132,2.183,39.3168478712251,0.2034814453125,55.9185976664178,8
4+
6,32,1,128,256,473.192594502983,1.62366666666666,41.6249986553418,0.144159505208333,59.040590405904,6
5+
4,48,1,128,256,411.965896564616,1.24299999999999,33.9852688224386,0.1177021484375,48.5599570042047,4
6+
2,96,1,128,256,305.354978576671,0.8385,24.7206425382519,0.08091015625,35.357488145113,2
7+
1,192,1,128,256,194.233687405159,0.659,15.732546705998,0.0635625,22.6802905912232,1
8+
12,16,2,128,256,542.868063529982,5.6595,72.3228012539645,0.331864583333333,100.041249647206,24
9+
8,24,2,128,256,525.313284346235,3.8995,72.0523229968335,0.222072265625,99.8261491218093,16
10+
6,32,2,128,256,521.6867408124,2.945,77.734407874991,0.1543984375,107.242599143548,12
11+
4,48,2,128,256,476.123810929111,2.15075,61.4768015649479,0.1301328125,86.1010678550408,8
12+
2,96,2,128,256,392.824770035245,1.3035,45.6750702061003,0.087576171875,64.502582622937,4
13+
1,192,2,128,256,265.010351966873,0.966,27.9582810025664,0.07153515625,39.8360910835624,2
14+
12,16,4,128,256,542.347674433062,11.3286666666666,132.965384586295,0.361004557291666,176.638012822355,48
15+
8,24,4,128,256,534.197014471598,7.66799999999999,122.135263742044,0.262018554687499,162.305670395857,32
16+
6,32,4,128,256,530.355025111731,5.79283333333333,130.851069474605,0.183422526041666,173.569128199333,24
17+
4,48,4,128,256,498.926336797732,4.105,110.286655292394,0.1450771484375,148.732721682926,16
18+
2,96,4,128,256,429.673419161075,2.3835,83.9516209685484,0.095296875,113.938135153178,8
19+
1,192,4,128,256,323.845667299177,1.581,45.6266987479392,0.08766796875,63.9360639360639,4
20+
12,16,8,128,256,537.144575586083,22.87675,238.711049155405,0.402173502604166,290.629286829283,96
21+
8,24,8,128,256,528.081343748855,15.5135,221.629401743664,0.28878662109375,271.024945411235,64
22+
6,32,8,128,256,525.56856057629,11.6906666666666,221.216197420452,0.216985026041666,272.993868301786,48
23+
4,48,8,128,256,501.229955699081,8.172,185.573035867483,0.1724404296875,234.115114218758,32
24+
2,96,8,128,256,436.004368890238,4.6975,122.285100101095,0.130841796875,160.656852234395,16
25+
1,192,8,128,256,334.53119895459,3.061,69.5557668795,0.115015625,94.508537148131,8
26+
12,16,16,128,256,512.073415226242,47.9935,289.993548021426,0.662094401041666,336.695969859573,192
27+
8,24,16,128,256,501.01982290772,32.703625,266.141862129395,0.48096533203125,313.036168058235,128
28+
6,32,16,128,256,497.243605625261,24.7124999999999,262.397967899881,0.365867838541666,309.680020833508,96
29+
4,48,16,128,256,473.416491535795,17.30425,217.319695888565,0.294501953125,264.170007846846,64
30+
2,96,16,128,256,415.396065258368,9.8605,158.520109837398,0.2018671875,199.649054396568,32
31+
1,192,16,128,256,315.416602494994,6.493,91.4653208878567,0.1749296875,119.824475865431,16
32+
12,16,32,128,256,460.984439877359,106.624666666666,253.363071621085,1.51563736979166,296.333214094798,384
33+
8,24,32,128,256,452.297016065001,72.44825,232.808757420772,1.09965185546875,274.63053499092,256
34+
6,32,32,128,256,446.791245308319,55.0066666666666,230.863107267069,0.831688151041666,273.86696680299,192
35+
4,48,32,128,256,420.62990827915,38.95225,198.650306397693,0.6443798828125,239.040569588857,128
36+
2,96,32,128,256,357.457026399537,22.923,144.129917880183,0.444123046875,177.451730760899,64
37+
1,192,32,128,256,238.639011885341,17.164,87.3422040259297,0.366375,110.74660225675,32

‎benchmarks/example_results/ampereone/ampere_optimized_llama.cpp_r1.2.6/Meta-Llama-3-8B-Instruct.Q8R16.gguf@PP128@TG256.csv‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
n_proc,n_threads,batch_size,prompt_size,output_tokens,pp_throughput_tps,pp_avg_latency_sec,tg_throughput_tps,tg_avg_latency_sec,pp+tg_throughput_tps,concurrency
2+
12,16,1,128,256,868.021350611679,1.77074999999999,27.1740426566711,0.441602864583333,39.9431365070559,12
3+
8,24,1,128,256,784.364093581668,1.310875,28.3653389752284,0.2820380859375,41.6006500101564,8
4+
6,32,1,128,256,787.101893074064,0.977333333333333,28.1191607108403,0.213383463541666,41.1855135676236,6
5+
4,48,1,128,256,650.136466700944,0.78775,24.0069197061409,0.166623046874999,35.0676925184356,4
6+
2,96,1,128,256,407.968386513475,0.6275,20.475449205116,0.0976835937499999,29.7386253630203,2
7+
1,192,1,128,256,236.598890942698,0.541,14.4657286545742,0.06912890625,21.0549402346748,1
8+
12,16,2,128,256,950.635734877784,3.23525,56.6431930272877,0.423711588541666,82.0900175474538,24
9+
8,24,2,128,256,908.274196566854,2.2565,55.4006177325249,0.28881689453125,80.0949041181608,16
10+
6,32,2,128,256,894.422906044505,1.71816666666666,53.5653765360743,0.224026041666666,77.8786188713684,12
11+
4,48,2,128,256,799.152589487598,1.284,50.5890123214361,0.1581396484375,72.9951289057859,8
12+
2,96,2,128,256,599.099468763164,0.855499999999999,43.9711719892178,0.090970703125,63.4029555023528,4
13+
1,192,2,128,256,342.245989304812,0.748,27.379679144385,0.073046875,39.489921842863,2
14+
12,16,4,128,256,942.192083763169,6.52216666666666,98.9327767577271,0.485189127604166,139.824915416243,48
15+
8,24,4,128,256,933.6581283838,4.39075,86.1509981609636,0.37145556640625,121.555049955485,32
16+
6,32,4,128,256,936.894776396972,3.27966666666666,87.3912377359475,0.274630208333333,124.493434916518,24
17+
4,48,4,128,256,853.328676570562,2.4005,82.807795698789,0.1932294921875,117.343723142153,16
18+
2,96,4,128,256,698.320343743605,1.4665,70.3716242682738,0.1136875,99.7953415846408,8
19+
1,192,4,128,256,463.768115942028,1.104,41.6870216577104,0.095953125,59.8410472183263,4
20+
12,16,8,128,256,927.048168921944,13.25625,182.861897571723,0.525006510416666,246.796545491062,96
21+
8,24,8,128,256,926.234457912476,8.846625,181.868635561571,0.351904296875,246.920526474429,64
22+
6,32,8,128,256,931.552074692618,6.596,179.461825901815,0.267479817708333,243.535707207504,48
23+
4,48,8,128,256,857.121630157963,4.77924999999999,156.145146951637,0.204940429687499,214.054280040413,32
24+
2,96,8,128,256,719.170636916915,2.848,123.13754535315,0.129939453125,169.46160635481,16
25+
1,192,8,128,256,481.882352941176,2.125,70.1369863013698,0.1140625,98.0686352753391,8
26+
12,16,16,128,256,852.734514233319,28.8215833333333,269.023699772012,0.713709309895833,345.556805399325,192
27+
8,24,16,128,256,849.700893221581,19.283,267.906591696701,0.47779296875,343.041393605661,128
28+
6,32,16,128,256,847.084190097491,14.5073333333333,273.190364528141,0.351421223958333,350.638233112028,96
29+
4,48,16,128,256,786.279129338369,10.419,240.676892425744,0.26592578125,311.009870918754,64
30+
2,96,16,128,256,658.160906097684,6.224,191.020607897768,0.167523437499999,249.108011676938,32
31+
1,192,16,128,256,432.980972515856,4.73,100.495608224152,0.1592109375,135.068589518114,16
32+
12,16,32,128,256,710.94680980739,69.13775,313.778103782508,1.22380859375,383.60740179972,384
33+
8,24,32,128,256,707.661078308765,46.3073749999999,294.790867147539,0.868453613281249,361.355967093316,256
34+
6,32,32,128,256,703.260600754351,34.9475,296.177462440666,0.648305989583333,363.555673900501,192
35+
4,48,32,128,256,649.992959465852,25.20725,249.901684113553,0.5122119140625,312.556435920589,128
36+
2,96,32,128,256,529.206227246949,15.481,173.77943934586,0.368320312499999,221.689007559219,64
37+
1,192,32,128,256,339.325656532184,12.071,98.100735276507,0.3261953125,128.566496123544,32

‎benchmarks/example_results/ampereone/llama.cpp_b3615/Meta-Llama-3-8B-Instruct.Q4_K_M.gguf@PP128@TG256.csv‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
n_proc,n_threads,batch_size,prompt_size,output_tokens,pp_throughput_tps,pp_avg_latency_sec,tg_throughput_tps,tg_avg_latency_sec,pp+tg_throughput_tps,concurrency,start,finish
2+
24,8,1,128,256,253.20046372915644,12.135249999999997,38.69025173804739,0.6203320312500001,53.48344601456635,24,1724334079.7983723,1724334257.7531338
3+
16,12,1,128,256,253.52587783504268,8.078812499999998,45.37226497470845,0.35267285156250006,61.28433778203363,16,1724334257.9896073,1724334361.316836
4+
12,16,1,128,256,251.31249011909875,6.112333333333333,44.42436957738335,0.2701429036458333,60.144880245382765,12,1724334362.093771,1724334441.261901
5+
8,24,1,128,256,247.8675851674121,4.13225,40.59404286689349,0.19708642578125,55.230937955088905,8,1724334442.1756241,1724334499.978329
6+
6,32,1,128,256,242.9533142226865,3.1618333333333335,41.02839965140533,0.14625390624999998,56.14308689507286,6,1724334500.2309978,1724334543.3615577
7+
4,48,1,128,256,225.78247842686523,2.26825,33.639986473321,0.118939453125,45.952252737390054,4,1724334544.2783625,1724334579.6622102
8+
3,64,1,128,256,214.97518039932388,1.787,31.202790511921123,0.09615885416666665,43.00272499906678,3,1724334580.3126905,1724334608.872495
9+
2,96,1,128,256,191.37310290729246,1.338,30.545743416433837,0.06553125,41.23932771304301,2,1724334609.3459954,1724334629.8076026
10+
1,192,1,128,256,132.64248704663214,0.965,21.02496714848883,0.0475625,29.22152043223499,1,1724334630.3672194,1724334645.238105
11+
24,8,2,128,256,262.49577991931454,23.40879166666667,81.78734770154088,0.5869134114583334,103.7831994189222,48,1724334645.3917124,1724334826.9875073
12+
16,12,2,128,256,260.66324577513916,15.715375000000002,79.39979845817122,0.40303100585937507,101.98695284099398,32,1724334827.572123,1724334950.9185264
13+
12,16,2,128,256,260.74447413923394,11.783083333333332,75.90944246869424,0.31617545572916667,97.99564038492213,24,1724334951.7005467,1724335048.2905056
14+
8,24,2,128,256,258.22943712149896,7.932250000000002,83.32954614579377,0.1920517578125,104.23275935193826,16,1724335048.7954853,1724335109.908087
15+
6,32,2,128,256,256.5891749455399,5.987333333333333,79.5234532484058,0.15091341145833334,101.78027123735477,12,1724335110.854442,1724335158.1402109
16+
4,48,2,128,256,245.3254663569985,4.1745,69.36340726100141,0.11535644531250001,89.37507273362039,8,1724335158.9118173,1724335195.0979106
17+
3,64,2,128,256,235.8756147123962,3.256,59.43066156773247,0.10097526041666666,77.83257887980542,6,1724335195.9452443,1724335227.4101338
18+
2,96,2,128,256,216.26212076609482,2.3675,59.44945548591053,0.06732421875,76.71944458318765,4,1724335227.9781172,1724335249.799204
19+
1,192,2,128,256,162.53968253968253,1.575,40.429564118761846,0.04946875,53.93637193623148,2,1724335249.996748,1724335265.9599679
20+
24,8,4,128,256,265.6376772935371,46.263083333333334,144.7827288856438,0.6630774739583335,168.01498571161622,96,1724335267.0267599,1724335490.443299
21+
16,12,4,128,256,264.6539255201115,30.9575,145.23363854559855,0.44068212890624997,168.87819962205805,64,1724335491.2533526,1724335639.6880503
22+
12,16,4,128,256,264.4493899924238,23.235166666666668,141.5457663974318,0.33914713541666663,164.73322012691037,48,1724335640.4046247,1724335754.851841
23+
8,24,4,128,256,262.10192399244164,15.630875,124.33171094495091,0.25738720703125,148.7165212339792,32,1724335755.5158155,1724335840.2709846
24+
6,32,4,128,256,260.5912149296376,11.789666666666667,125.7524185906364,0.19086328125,149.93167176416998,24,1724335840.604125,1724335904.15949
25+
4,48,4,128,256,254.81472598528924,8.03775,107.03866832942929,0.1494833984375,131.68442034421415,16,1724335904.6698062,1724335953.2111015
26+
3,64,4,128,256,249.30439289990755,6.162,103.23486482315428,0.1162421875,127.88277412371991,12,1724335953.7153456,1724335991.5463428
27+
2,96,4,128,256,232.55273275185664,4.403499999999999,91.96361516113048,0.086998046875,114.17527688991304,8,1724335991.7615826,1724336020.46686
28+
1,192,4,128,256,183.9080459770115,2.784,61.29167414856048,0.06526171875,78.80560258580884,4,1724336020.7816617,1724336041.989612
29+
24,8,8,128,256,262.50853923875883,93.62595833333332,186.55635404600494,1.0291969401041667,203.61338643129758,192,1724336042.810995,1724336408.9396646
30+
16,12,8,128,256,261.48092312812935,62.662187499999995,179.96241765060614,0.7112668457031249,199.21936746878077,128,1724336409.181211,1724336658.8075428
31+
12,16,8,128,256,262.91219764803964,46.742,176.17135182465427,0.5449388020833332,195.04349114304458,96,1724336659.4405437,1724336851.0648277
32+
8,24,8,128,256,259.9820602180153,31.513375,168.26239946839615,0.38036230468749993,189.58428153759516,64,1724336851.6313999,1724336983.4608665
33+
6,32,8,128,256,259.4447228251579,23.684166666666666,160.52023922108944,0.29903190104166666,183.38473783703117,48,1724336983.7668731,1724337086.3126478
34+
4,48,8,128,256,254.94851640587902,16.06675,149.31307066555968,0.21431640625,172.822142837051,32,1724337086.8721328,1724337159.862079
35+
3,64,8,128,256,249.92825097923446,12.292,137.4061792115868,0.17467578125000002,160.3926277867697,24,1724337160.940714,1724337220.1721318
36+
2,96,8,128,256,236.0568959829584,8.676,120.58794533528942,0.13268359375,144.00562521973515,16,1724337221.0059676,1724337265.4647002
37+
1,192,8,128,256,191.36609979443094,5.351,79.64842686578773,0.10044140625,98.89260880762298,8,1724337266.0485623,1724337299.095685
38+
24,8,16,128,256,254.5893375396671,193.07433333333336,191.7644035299183,2.0025024414062504,205.968873356297,384,1724337300.099435,1724338020.1539094
39+
16,12,16,128,256,254.2628430014775,128.8820625,184.1624285649668,1.3900893554687497,201.14338095374504,256,1724338020.819572,1724338512.418089
40+
12,16,16,128,256,254.24955581411595,96.66608333333333,184.21557164290846,1.0422841796875,200.23573760340682,192,1724338513.3095915,1724338884.1088865
41+
8,24,16,128,256,253.71071857003383,64.582375,172.34809861392793,0.74268505859375,192.11331683922938,128,1724338884.6845584,1724339142.7994077
42+
6,32,16,128,256,253.0907914660322,48.55516666666667,173.15947998516668,0.5544049479166667,192.5716972261401,96,1724339142.9391003,1724339336.4508188
43+
4,48,16,128,256,249.21709386631983,32.8715,165.25611057383236,0.3872783203125,185.66420887223498,64,1724339337.1388545,1724339471.4487627
44+
3,64,16,128,256,245.31254129994232,25.045666666666666,158.4039376236316,0.30302734375,179.05054253324653,48,1724339472.2731543,1724339577.0808818
45+
2,96,16,128,256,231.6909941911096,17.6795,136.90584191720038,0.233740234375,157.84808663146944,32,1724339577.3750696,1724339657.0617642
46+
1,192,16,128,256,186.50396138785175,10.981,93.06764218036399,0.17191796875,111.72534186790806,16,1724339657.4612248,1724339714.184876
47+
12,16,32,128,256,240.00769185542717,204.7976666666667,150.02250416967684,2.5596298828125,170.93890269688762,384,1724341538.2196314,1724342403.3936372
48+
8,24,32,128,256,239.1554429112951,137.021625,147.06983144573388,1.740673828125,167.9701049300552,256,1724342404.0835724,1724342991.5957937
49+
6,32,32,128,256,240.09918854541434,102.36383333333333,145.09067012896256,1.3233144531249998,166.12738956836802,192,1724342992.6787407,1724343438.55426
50+
4,48,32,128,256,235.2577992061292,69.64475,142.39669954076032,0.898900390625,163.57010931962262,128,1724343439.1234324,1724343741.5138347
51+
3,64,32,128,256,231.6361368854605,53.04933333333333,135.62939562093007,0.7078151041666668,156.91816537192722,96,1724343742.4260345,1724343979.2283869
52+
2,96,32,128,256,217.99184644192724,37.5805,118.5114510841291,0.540076171875,138.61877580488687,64,1724343979.6653004,1724344158.770979
53+
1,192,32,128,256,169.52940689540995,24.161,83.05284074779999,0.385296875,100.06759122779873,32,1724344159.845426,1724344284.2662663

0 commit comments

Comments
 (0)