add m6a m6i m7a m7i

Author: MarcelWilnicki

benchmarks/example_results/m6a/llama.cpp_1dc04b2/Llama-3.2-1B-Instruct-Q4_K_M.gguf@PP512@TG256.csv

@@ -0,0 +1,43 @@
+n_proc,n_threads,batch_size,prompt_size,output_tokens,pp_throughput_tps,pp_avg_latency_sec,tg_throughput_tps,tg_avg_latency_sec,pp+tg_throughput_tps,concurrency,avg_power_watts
+12,8,1,512,256,2237.7115855668594,2.7495833333333337,74.20530103847251,0.19288639322916667,125.2974045926067,12,159.54607142857142
+8,12,1,512,256,2110.671116479293,1.9419999999999997,49.660443119290605,0.168435546875,105.51262235960844,8,165.82936363636364
+6,16,1,512,256,1949.1888062727535,1.5773333333333335,38.306796573803936,0.16036458333333334,90.12497799683155,6,163.95197916666666
+4,24,1,512,256,1716.3815213089779,1.1955,34.303419776646344,0.1182822265625,86.99345849970267,4,168.12953124999999
+3,32,1,512,256,1264.442814914618,1.2223333333333335,30.93005761615281,0.09760807291666666,79.88350322446432,3,172.92423076923075
+2,48,1,512,256,1028.1787828443312,0.996,31.932606988037396,0.062650390625,88.77586406195816,2,166.18714285714285
+1,96,1,512,256,464.1885766092475,1.103,48.39319470699433,0.0206640625,120.13139371187236,1,131.70166666666668
+12,8,2,512,256,2247.933617116949,5.46775,153.67389748299865,0.18236393229166661,256.74169823935813,24,175.0345652173913
+8,12,2,512,256,2131.005832125184,3.8453749999999998,123.71599107983299,0.14307373046875,223.33290925283075,16,176.46894230769232
+6,16,2,512,256,1996.2653701640252,3.0785,100.76405715417147,0.12832877604166668,200.7318348144276,12,171.46197674418605
+4,24,2,512,256,1748.4002996496179,2.3434999999999997,82.34002449209753,0.09997460937500001,188.52408714329547,8,176.0860344827586
+3,32,2,512,256,1409.3218622004622,2.1943333333333332,68.18491441407517,0.08961067708333333,159.03364969801552,6,178.26365384615386
+2,48,2,512,256,1139.1219367368903,1.7985,61.59591245949126,0.065185546875,157.17574827321565,4,174.39125
+1,96,2,512,256,491.12709832134294,2.085,43.64132287759973,0.045828125,111.16740247521169,2,167.355
+12,8,4,512,256,2103.006391038348,11.68725,287.15163650496936,0.18903808593750002,467.62735944793997,48,202.2666447368421
+8,12,4,512,256,1976.7844198883042,8.28875,235.4162179799681,0.14611669921875,419.77248659173983,32,204.57663636363637
+6,16,4,512,256,1905.2972874213203,6.450500000000001,189.5040515310117,0.13347005208333332,369.2455627228655,24,193.32606382978724
+4,24,4,512,256,1647.7450522546194,4.97225,168.65233859714164,0.0973720703125,356.3495055534611,16,197.09951612903225
+3,32,4,512,256,1157.933487961232,5.497,131.66539708436355,0.09125390625,314.7648485262475,12,200.4603846153846
+2,48,4,512,256,1104.9038064559904,3.753,118.42603280902823,0.06880468749999999,263.3744855967078,8,199.08075
+1,96,4,512,256,479.51299461484433,4.271,104.4152136229224,0.03830859375,218.21281432021595,4,195.28545454545454
+12,8,8,512,256,1800.1796217140618,27.30516666666667,549.2657934032587,0.18684895833333334,814.4760389739511,96,260.996091954023
+8,12,8,512,256,1682.6603434466335,19.477874999999997,422.00485192268326,0.15539453124999997,712.8023667266084,64,253.82659090909092
+6,16,8,512,256,1619.0684262597451,15.185000000000002,386.2090879352144,0.12805533854166665,664.6113905565471,48,245.86086538461538
+4,24,8,512,256,1450.2733537447748,11.29875,347.48104325064253,0.09323632812500002,631.1410154343974,32,245.63833333333332
+3,32,8,512,256,1153.5038315971522,11.022666666666666,283.2663489240688,0.08482421875000001,505.8594286027939,24,236.14742424242422
+2,48,8,512,256,1016.0261507934036,8.064,249.86704571471688,0.06403515625,499.0253411306043,16,223.4502380952381
+1,96,8,512,256,427.3789649415693,9.584,155.7059226032084,0.05137890625,270.2203456920438,8,196.50275
+12,8,16,512,256,1373.8623043178657,71.55666666666666,688.2993850901098,0.27904687499999997,1016.0270102666575,192,283.74334507042255
+8,12,16,512,256,1270.6612111585628,51.57937500000001,564.720997313507,0.22680371093749999,875.0500707666836,128,283.46995412844035
+6,16,16,512,256,1204.0244429999466,40.994,532.647711989758,0.18025325520833335,803.8903547986131,96,279.7723033707865
+4,24,16,512,256,1099.5692965998803,29.80225,451.3750930081004,0.14180859375,741.0221619176843,64,275.8556349206349
+3,32,16,512,256,907.0032832250583,27.371666666666666,360.5485577437514,0.13441536458333334,521.2522270297785,48,257.17198529411763
+2,48,16,512,256,756.7918603030894,21.6735,319.165359505396,0.1002734375,508.31471829238023,32,258.5238888888889
+1,96,16,512,256,303.53106821297564,26.989,136.77040203018566,0.116984375,215.8174824806365,16,204.4125
+12,8,32,512,256,923.8950551217331,212.809,438.93344784774683,0.874923828125,668.0863017298404,384,285.8369178082192
+8,12,32,512,256,816.0560488193221,160.63675,381.62882588318837,0.6711562500000001,576.9031012414943,256,284.56001479289944
+6,16,32,512,256,831.8886939268943,118.20333333333333,360.28223835071316,0.5331217447916666,564.3773538687651,192,284.4968410852713
+4,24,32,512,256,728.1976638156776,89.99799999999999,322.81640142174484,0.39661328124999995,509.5953469565488,128,283.9021578947369
+3,32,32,512,256,629.3676938874587,78.85166666666667,241.46770297780114,0.40149869791666665,354.9718104390446,96,259.8159024390244
+2,48,32,512,256,508.5679383219722,64.4435,218.63810759632145,0.2927265625,349.66706505036706,64,268.47467391304343
+1,96,32,512,256,222.45756958587913,73.65,106.33575202170329,0.30093359375,163.0908692738023,32,221.94016891891889

benchmarks/example_results/m6a/llama.cpp_1dc04b2/Llama-3.2-1B-Instruct-Q8_0.gguf@PP512@TG256.csv

@@ -0,0 +1,43 @@
+n_proc,n_threads,batch_size,prompt_size,output_tokens,pp_throughput_tps,pp_avg_latency_sec,tg_throughput_tps,tg_avg_latency_sec,pp+tg_throughput_tps,concurrency,avg_power_watts
+12,8,1,512,256,2319.040136448242,2.65375,54.46486282470201,0.2640201822916666,90.93241243216576,12,157.09
+8,12,1,512,256,1531.053179874824,2.6765000000000003,46.718275943246105,0.1996201171875,82.35924932975873,8,163.22270833333334
+6,16,1,512,256,1436.4835227221383,2.1413333333333333,34.82366531512506,0.184099609375,72.74908826826226,6,160.56541666666666
+4,24,1,512,256,1329.1262660637776,1.54175,25.800446057630275,0.15726171875,63.375487384729645,4,170.05666666666667
+3,32,1,512,256,978.0227251144652,1.5723333333333336,20.399906422353094,0.14826822916666668,53.877092881863256,3,167.6115
+2,48,1,512,256,843.0483420799754,1.274,18.926663450523026,0.105677734375,53.30002082032064,2,169.5119230769231
+1,96,1,512,256,419.3284193284193,1.221,29.364533149805002,0.0340546875,77.2713552671295,1,153.44857142857146
+12,8,2,512,256,2341.1133865145084,5.252083333333334,108.67855937026965,0.2598740234375,181.45661462127626,24,164.86550505050505
+8,12,2,512,256,1549.5739353764898,5.287750000000001,78.05061297653945,0.21859521484375,159.0432554166343,16,175.44567567567566
+6,16,2,512,256,1477.1764108553443,4.162,57.93925896732674,0.20876822916666668,138.48026325675045,12,175.38730158730158
+4,24,2,512,256,1333.2272371091772,3.0725,42.07219303517002,0.191154296875,108.43820046241551,8,173.09629629629632
+3,32,2,512,256,1122.9758345436605,2.7449999999999997,36.6011962891607,0.16397265625,100.75875188594668,6,177.5974418604651
+2,48,2,512,256,904.9750340374096,2.268,36.20867227605766,0.110689453125,96.0900844541758,4,171.55568965517242
+1,96,2,512,256,417.1079429735234,2.455,54.139790631278416,0.03694140625,128.94560107454666,2,185.25444444444443
+12,8,4,512,256,2187.7651803000654,11.235,218.31190901802032,0.25323144531249997,353.8966649386556,48,187.13628712871287
+8,12,4,512,256,1477.5041701197244,11.089749999999999,175.224312670163,0.19775439453125002,321.49024122233266,32,197.2436301369863
+6,16,4,512,256,1415.8094724524583,8.679833333333333,125.0373469192628,0.1942154947916667,271.47801752706386,24,190.51484615384615
+4,24,4,512,256,1287.7727914682455,6.361750000000001,103.1883738581665,0.1556748046875,242.20921293832416,16,192.7820833333333
+3,32,4,512,256,1070.631584347729,5.888999999999999,78.86171398327403,0.15216666666666667,199.64472943113384,12,193.02197674418605
+2,48,4,512,256,993.420479844778,4.134,71.08611125213977,0.11323242187499999,174.65957870199279,8,187.31625
+1,96,4,512,256,491.71668667466986,4.165,84.22437900970554,0.0474921875,188.20069840102923,4,185.16148148148147
+12,8,8,512,256,1874.4743299358252,26.223,412.3256956075942,0.2598141276041667,619.0428211586901,96,230.20745689655172
+8,12,8,512,256,1299.856221489251,25.212,349.1440893932651,0.19489160156250002,554.5501726199878,64,242.77959064327484
+6,16,8,512,256,1249.9479831740146,19.670000000000005,298.9469559309155,0.16689973958333335,505.9219103822137,48,234.07757142857145
+4,24,8,512,256,1145.1520889371523,14.308,258.4860736571353,0.12618261718749998,474.06492930306126,32,232.71551020408162
+3,32,8,512,256,970.8729397114903,12.729,189.1478633658188,0.12691666666666665,396.25075242927164,24,224.7970930232558
+2,48,8,512,256,875.7460185055577,9.3555,187.1534330297515,0.08549609375,389.95906191488683,16,215.8994642857143
+1,96,8,512,256,373.0758721195009,10.979,144.56130443989554,0.05533984375,244.3330947267955,8,194.75931818181817
+12,8,16,512,256,1418.1146515256119,69.32241666666665,591.622386149424,0.33118131510416665,852.5784461674559,192,278.13679411764707
+8,12,16,512,256,1046.721832514154,62.62175000000001,470.47813680912225,0.27320361328125,702.8039521283441,128,282.15459854014597
+6,16,16,512,256,1004.5508479645216,48.94633333333334,438.2981184237237,0.21944791666666666,679.1639415238078,96,280.142
+4,24,16,512,256,918.6581250068461,35.67375,387.84803350392013,0.1654921875,606.522785325584,64,275.4741666666667
+3,32,16,512,256,799.7883324704474,30.880666666666666,300.01987222851875,0.16109505208333333,461.3478505725549,48,256.1535714285714
+2,48,16,512,256,667.5014549542846,24.595,252.1746644283902,0.1269296875,418.5072288541117,32,247.24
+1,96,16,512,256,323.80726510929287,25.299,152.5000930786701,0.10491796875,235.59185551593237,16,212.66744897959182
+12,8,32,512,256,943.193250573753,208.45758333333333,449.61758210568127,0.8542265625,680.8039115198692,384,285.83112790697675
+8,12,32,512,256,716.6318189168906,182.90775,353.0842664638377,0.72538671875,525.966886301073,256,285.4397708894879
+6,16,32,512,256,721.5994704893152,136.26200000000003,347.1564739146902,0.55344921875,513.9487222384876,192,284.08271126760565
+4,24,32,512,256,645.6566172160015,101.50525,315.332583094978,0.4059853515625,475.3601323023806,128,283.33708333333334
+3,32,32,512,256,569.7136905221678,86.907,235.7797921729392,0.4108841145833333,338.9294448632845,96,259.7079439252336
+2,48,32,512,256,481.6662353138124,68.039,222.01295712351043,0.28828125000000004,343.6505883422243,64,270.9449285714286
+1,96,32,512,256,223.10279559350192,73.437,109.18881454429132,0.2930703125,165.5361942032695,32,221.10986301369863

benchmarks/example_results/m6a/llama.cpp_1dc04b2/Llama-3.2-3B-Instruct-Q4_K_M.gguf@PP512@TG256.csv

@@ -0,0 +1,42 @@
+n_proc,n_threads,batch_size,prompt_size,output_tokens,pp_throughput_tps,pp_avg_latency_sec,tg_throughput_tps,tg_avg_latency_sec,pp+tg_throughput_tps,concurrency,avg_power_watts
+12,8,1,512,256,803.8705393271796,7.644916666666666,38.56100858972805,0.3749430338541668,61.7810312927359,12,163.5103355704698
+8,12,1,512,256,768.9100806124642,5.328625,32.293317640878996,0.27739599609375,57.91363854876566,8,166.8021327014218
+6,16,1,512,256,689.558590760419,5.157666666666667,25.82707364243524,0.24990364583333333,51.655718225231496,6,167.72727272727272
+4,24,1,512,256,662.3715277946694,3.09275,20.06391146080426,0.20154785156250002,48.54307565893433,4,172.80766129032259
+3,32,1,512,256,361.48826455966537,5.237666666666667,17.52430427261226,0.17183723958333333,44.39905189524599,3,181.38049019607845
+2,48,1,512,256,278.1443571359711,5.6295,16.33674204342288,0.122423828125,38.07823888145173,2,176.24620253164557
+1,96,1,512,256,246.98504582730342,2.073,18.4517803084907,0.0541953125,48.15952843795071,1,164.486
+12,8,2,512,256,828.0740383266753,14.840583333333335,79.81757849569847,0.3631842447916666,120.69541302426087,24,178.89588815789475
+8,12,2,512,256,793.9022590325959,10.319375,65.51449706864861,0.27444140624999996,110.7356241044635,16,180.3059090909091
+6,16,2,512,256,769.488703757948,7.987166666666667,63.68912620201702,0.216630859375,106.597575645415,12,175.33453488372095
+4,24,2,512,256,713.8291508443613,5.7387500000000005,51.7023614304124,0.1638955078125,102.72701432894713,8,184.66898305084746
+3,32,2,512,256,358.47536056312117,9.658,51.47996552273075,0.12342317708333335,93.40982343759501,6,187.85857142857142
+2,48,2,512,256,501.62882707173935,4.083,34.09152037829631,0.118390625,82.37470839031454,4,176.99418918918917
+1,96,2,512,256,318.8044831880448,3.212,38.2803738317757,0.05224609375,92.602640622174,2,174.7221875
+12,8,4,512,256,797.0001140106879,30.836916666666664,151.17633513556382,0.3715859375000001,218.26966345356797,48,210.10839285714286
+8,12,4,512,256,766.5274703990373,21.375749999999996,131.60700042386998,0.26971484375000004,207.2577312632298,32,212.69097457627117
+6,16,4,512,256,743.4012326236221,16.566,124.12278577962587,0.21879427083333333,195.43020728410116,24,208.74048128342247
+4,24,4,512,256,704.1662529793396,11.634,106.83933662352804,0.15945117187500002,189.80537534754404,16,213.730078125
+3,32,4,512,256,547.3516327834496,11.896,102.7853262913801,0.119265625,181.17836711423908,12,217.0421
+2,48,4,512,256,531.3249919138568,7.713,65.44399419654755,0.12264257812500001,149.565471408749,8,198.76175
+1,96,4,512,256,140.53386399505936,14.573,36.287607640242385,0.11023046875,71.7891194615816,4,178.8165476190476
+12,8,8,512,256,713.7630410222706,68.86591666666665,260.6104216533761,0.40363346354166657,350.01898974553745,96,261.70238095238096
+8,12,8,512,256,683.7332299678702,47.925875000000005,227.60235494454247,0.29707275390625,331.0188770734138,64,263.64179054054057
+6,16,8,512,256,649.1777761147957,38.259499999999996,222.1382965837935,0.23074283854166666,324.0050625791028,48,262.9516371681416
+4,24,8,512,256,633.6967037304801,25.856,191.82608828744168,0.1727021484375,306.0256266577011,32,261.5723417721519
+3,32,8,512,256,409.7976644551353,30.189666666666668,151.00344645498882,0.15984114583333334,250.00678186799772,24,238.8022602739726
+2,48,8,512,256,499.6431619057548,16.4135,110.60671695073152,0.14468945312499998,229.8068111686709,16,232.1288679245283
+1,96,8,512,256,174.18668934722518,23.515,92.77463193657985,0.08623046875,134.76639613950428,8,193.66933333333336
+12,8,16,512,256,580.4949504304288,169.35166666666666,315.09527263358217,0.6105247395833334,437.4977747712465,192,283.18780089153046
+8,12,16,512,256,551.2492298742482,118.89824999999999,277.3952649686004,0.4622607421875,400.6210800435245,128,282.67573469387753
+6,16,16,512,256,542.5990644781376,90.70933333333333,262.9185312948969,0.3656783854166667,391.28981069190064,96,282.14398936170215
+4,24,16,512,256,519.090666738386,63.12875,235.03765315442294,0.2727998046875,357.83343040186367,64,280.436336996337
+3,32,16,512,256,459.21365025951775,53.797,194.22419268617813,0.24960807291666667,278.9662870331833,48,264.31988593155893
+2,48,16,512,256,409.78526533080185,39.982,160.68377301104346,0.199150390625,269.93245098577626,32,267.9241666666667
+1,96,16,512,256,162.73987842186816,50.338,75.5259712721037,0.21184765625,117.50867831425539,16,214.55913461538464
+8,12,32,512,256,388.45615818674037,337.45487499999996,195.4588447928328,1.31003515625,288.8206285339056,256,285.7831029411765
+6,16,32,512,256,394.53056153253016,249.25683333333333,193.6356830938425,0.9916015625,288.41516010350836,192,284.93374755381603
+4,24,32,512,256,372.6411034275528,175.87525,171.02720034486433,0.7489306640625,263.39918491585235,128,284.33336912751673
+3,32,32,512,256,325.05198564510107,151.98066666666668,134.5445186233997,0.7166263020833333,200.7340208880128,96,268.8047683923706
+2,48,32,512,256,285.4417899303714,114.7975,115.14821053972506,0.555806640625,191.06334182037278,64,278.57249027237356
+1,96,32,512,256,135.55507752386941,120.866,56.796594422951586,0.5634140625,92.70463975858166,32,232.80075614366731