Merge pull request #10 from AmpereComputingAI/marcel/update-csv

add csv files

Author: MarcelWilnicki

README.md

@@ -49,6 +49,9 @@ For example:
 ./llama-quantize llama-2-7b-f16.gguf llama-2-7b-Q8R16.gguf Q8R16
 ```
 
+## Benchmark Results
+Benchmark results conducted by our Team can be found in **benchmarks/example_results**, with data selectable by machine type and software.
+
 ## Support
 
 Please contact us at <ai-support@amperecomputing.com>

benchmarks/example_results/a1/ampere_optimized_llama.cpp_r2.0.0/Llama-3.2-1B-Instruct-Q4_K_4.gguf@PP512@TG256.csv

@@ -0,0 +1,67 @@
+n_proc,n_threads,batch_size,prompt_size,output_tokens,pp_throughput_tps,pp_avg_latency_sec,tg_throughput_tps,tg_avg_latency_sec,pp+tg_throughput_tps,concurrency,start,finish
+10,8,1,512,256,1891.9868612029534,2.7067000000000005,294.42038195695295,0.033966015625000005,664.3023959865063,10,1730385886.666355,1730385899.4342008
+8,10,1,512,256,1860.0794862469354,2.20225,299.1809517743992,0.02673974609375,674.2756804214223,8,1730385899.6600862,1730385909.8421202
+6,12,1,512,256,1700.8312054558517,1.8063333333333331,292.5457456811743,0.020509765625,651.4915877279795,6,1730385910.6681628,1730385918.7391348
+5,16,1,512,256,1900.9019731952362,1.3468,289.28221307564206,0.017284374999999998,664.01521701539,5,1730385919.6714282,1730385926.425
+4,20,1,512,256,1755.901125892684,1.1665,274.0717174185161,0.0145947265625,624.6441642944286,4,1730385926.6775808,1730385932.457447
+3,24,1,512,256,1673.2145273219971,0.918,245.89545088028814,0.012200520833333334,567.4876847290641,3,1730385932.6776125,1730385937.548049
+2,32,1,512,256,1444.9344818252744,0.709,211.39568502304775,0.009460937499999999,488.54961832061065,2,1730385937.6848118,1730385941.596675
+2,40,1,512,256,1678.3955754825893,0.6105,217.82679286059482,0.009181640625000001,516.8236877523553,2,1730385942.6808684,1730385946.4201593
+1,48,1,512,256,1034.3434343434344,0.495,130.47910295616717,0.0076640625,312.5763125763126,1,1730385946.6828535,1730385949.802194
+1,64,1,512,256,1280.0,0.4,130.34623217922606,0.007671875,324.8730964467005,1,1730385950.6865938,1730385953.738844
+1,80,1,512,256,1302.7989821882952,0.393,128.2565130260521,0.007796875,321.4734198409376,1,1730385954.6930323,1730385957.7860413
+10,8,2,512,256,1835.918494740859,5.578,433.57472829858057,0.04612929687499999,879.7250859106529,20,1730385958.699984,1730385977.3860748
+8,10,2,512,256,1788.7335321621758,4.5802499999999995,451.2645997546241,0.0354560546875,896.2147181095472,16,1730385977.702211,1730385992.517355
+6,12,2,512,256,1614.456712542764,3.806166666666666,422.4906765515712,0.028402994791666664,828.9260658391797,12,1730385992.7104466,1730386004.817146
+5,16,2,512,256,1815.39632662427,2.8204000000000002,422.72158167422884,0.02365625,863.697705802969,10,1730386005.7136688,1730386015.54644
+4,20,2,512,256,1730.557832797799,2.367,389.67292448064734,0.0205302734375,804.1884816753926,8,1730386015.7150686,1730386024.2302253
+3,24,2,512,256,1593.9308760635324,1.9273333333333333,358.71123515908715,0.0167265625,741.1935016889174,6,1730386024.7208672,1730386031.7650955
+2,32,2,512,256,1401.6939357922843,1.4615,284.44479561085876,0.0140625,604.4864226682408,4,1730386032.7284935,1730386038.579139
+2,40,2,512,256,1615.1580288193613,1.268,295.35631403604884,0.013542968749999999,648.5117162761242,4,1730386038.7268703,1730386044.2284496
+1,48,2,512,256,986.5125240847784,1.038,194.5288753799392,0.01028125,418.5286103542234,2,1730386044.7262943,1730386049.0563223
+1,64,2,512,256,1200.4689331770223,0.853,204.96397117694153,0.0097578125,458.37063563115487,2,1730386049.7372608,1730386053.7846491
+1,80,2,512,256,1294.5638432364096,0.791,203.65950676213208,0.0098203125,464.75037821482607,2,1730386054.7348025,1730386058.7301395
+10,8,4,512,256,1661.7891086517595,12.3245,493.8360931655053,0.08100195312500001,923.9653512993261,40,1730386059.740134,1730386094.2170818
+8,10,4,512,256,1605.5412359116729,10.205499999999999,502.11501288310643,0.0637314453125,920.4494382022472,32,1730386094.7455428,1730386122.539461
+6,12,4,512,256,1462.039830323868,8.405000000000001,474.63751436806984,0.05056510416666667,861.0669905633935,24,1730386122.7557993,1730386145.1628454
+5,16,4,512,256,1619.4044331343848,6.323600000000001,471.9125977232286,0.042381249999999995,890.2799513128151,20,1730386145.7656453,1730386163.9540677
+4,20,4,512,256,1558.8357553933051,5.2555,470.22367461238326,0.0340263671875,877.8396913844836,16,1730386164.7632396,1730386179.6384294
+3,24,4,512,256,1427.6887263999065,4.303666666666667,396.6137424281863,0.030256510416666663,760.6470782436448,12,1730386179.7642553,1730386192.7303958
+2,32,4,512,256,1268.309277255304,3.2295,369.14204515790505,0.021671875,699.6128444545661,8,1730386193.7691615,1730386203.2793539
+2,40,4,512,256,1436.6988659916822,2.8515,386.560966402416,0.0206953125,750.3663898387885,8,1730386203.7782893,1730386212.747093
+1,48,4,512,256,895.1048951048951,2.288,243.63549845348558,0.01641796875,473.27068248343863,4,1730386213.7818983,1730386220.930257
+1,64,4,512,256,1122.1917808219177,1.825,259.043764229699,0.01544140625,531.6718587746625,4,1730386221.7801135,1730386228.2610428
+1,80,4,512,256,1158.3710407239819,1.768,268.2735132302856,0.01491015625,550.0447627573859,4,1730386228.7888181,1730386235.0772247
+10,8,8,512,256,1372.8581316948985,29.836600000000004,555.8013646759655,0.14393671875,917.686068916073,80,1730386235.7970197,1730386303.9549153
+8,10,8,512,256,1348.0104722128306,24.3095,543.0904481134129,0.11784521484375,896.6397898500493,64,1730386304.812397,1730386360.732647
+6,12,8,512,256,1230.087375272673,19.979333333333333,529.3902748269998,0.09067057291666665,851.2642881884309,48,1730386361.8198507,1730386406.1895456
+5,16,8,512,256,1344.7793565342963,15.2296,545.5179983740175,0.073325,900.3516998827666,40,1730386406.8292542,1730386441.8892336
+4,20,8,512,256,1252.5262397427766,13.081999999999999,501.4393281335627,0.06381738281249999,830.073968993819,32,1730386442.8321297,1730386473.3043616
+3,24,8,512,256,1163.9482096503916,10.558,422.16286859154434,0.056850260416666666,730.356223005904,24,1730386473.8386202,1730386499.9238372
+2,32,8,512,256,1032.2435111167506,7.9365000000000006,351.46788962387217,0.0455234375,624.9936422358984,16,1730386500.8422837,1730386521.266309
+2,40,8,512,256,1183.8696408173628,6.92,380.2808545226111,0.04207421875,692.2816901408451,16,1730386521.8505192,1730386540.3670537
+1,48,8,512,256,738.5503065272268,5.546,209.90058419596187,0.03811328125,401.48990394040385,8,1730386540.854028,1730386556.8428068
+1,64,8,512,256,904.1942604856512,4.53,235.70031073771435,0.03394140625,464.7855359709509,8,1730386557.8573027,1730386571.751055
+1,80,8,512,256,964.2184557438794,4.248,245.240091007065,0.03262109375,487.6577506151282,8,1730386572.861165,1730386586.1270695
+10,8,16,512,256,1007.3989862125478,81.3194,388.0627075345009,0.41230703124999996,654.8117043329053,160,1730386586.8732574,1730386775.7601094
+8,10,16,512,256,992.2192724968106,66.05100000000002,362.3106462417802,0.35329931640625006,625.3355555272834,128,1730386775.9033217,1730386934.2871184
+6,12,16,512,256,904.7555935356597,54.3265,390.935258009347,0.24557031249999997,627.1947733768885,96,1730386934.9369192,1730387053.4930768
+5,16,16,512,256,982.628261678772,41.6848,417.35161018780065,0.19168593750000001,674.6532848719103,80,1730387053.9517663,1730387145.9415977
+4,20,16,512,256,945.6942582242185,34.65025,386.9707738210735,0.1653916015625,636.1236216803851,64,1730387146.964929,1730387225.0779867
+3,24,16,512,256,848.2744186815962,28.972333333333335,336.483134289874,0.14265494791666666,559.6053130929791,48,1730387225.9821496,1730387292.6475954
+2,32,16,512,256,740.6885233505043,22.119999999999997,291.73281037304474,0.109689453125,489.0940932972457,32,1730387292.9863124,1730387343.99534
+2,40,16,512,256,851.0888757235361,19.251,327.7106795765708,0.09765429687499999,551.6622146400592,32,1730387344.9934084,1730387390.317172
+1,48,16,512,256,541.7989417989419,15.12,200.16615354542344,0.07993359375,345.3334457465644,16,1730387391.0043426,1730387427.2607112
+1,64,16,512,256,638.1056239289609,12.838,228.83959997765237,0.06991796875,399.77876825975204,16,1730387428.0086324,1730387459.408464
+1,80,16,512,256,685.4082998661312,11.952,242.53908100426335,0.06596875,426.0748959778085,16,1730387460.0146415,1730387489.5230558
+10,8,32,512,256,651.3602922412,251.539,234.846285510983,1.36260859375,407.7237910216487,320,1730387490.0264173,1730388094.0301788
+8,10,32,512,256,634.6367037330324,206.532375,231.83600680173316,1.1042421875,399.9259577186027,256,1730388094.1477003,1730388586.8601549
+6,12,32,512,256,590.6730442864924,166.42816666666667,239.43464301539709,0.801919921875,395.30743989533966,192,1730388587.2450256,1730388961.2400563
+5,16,32,512,256,627.6575382463196,130.5182,235.06801852303698,0.6806625000000001,401.1936556029332,160,1730388962.2900596,1730389269.4844882
+4,20,32,512,256,605.4088498295591,108.25175,235.4046150952486,0.543771484375,395.78703256353276,128,1730389270.3482082,1730389519.5924134
+3,24,32,512,256,551.1317837735581,89.18566666666668,215.7960704453041,0.44486588541666666,362.16271975714346,96,1730389520.3980958,1730389724.7842875
+2,32,32,512,256,468.7597960640328,69.904,191.94231836574858,0.33343359375000003,316.14492548545405,64,1730389725.4273415,1730389881.6655002
+2,40,32,512,256,492.2337390325106,66.57,214.9314963806475,0.29776953125,344.0378531231627,64,1730389882.437067,1730390026.0340118
+1,48,32,512,256,355.671333984587,46.065,142.34825974387044,0.22480078125,237.18802478429555,32,1730390026.4663146,1730390130.7651255
+1,64,32,512,256,407.2582649763858,40.23,160.48898989107437,0.199390625,269.2552095887109,32,1730390131.4798641,1730390223.443177
+1,80,32,512,256,439.8625429553265,37.248,177.37360614918262,0.18041015625,294.5597065909173,32,1730390224.4896095,1730390308.6187854

benchmarks/example_results/a1/ampere_optimized_llama.cpp_r2.0.0/Llama-3.2-1B-Instruct-Q8R16.gguf@PP512@TG256.csv

@@ -0,0 +1,67 @@
+n_proc,n_threads,batch_size,prompt_size,output_tokens,pp_throughput_tps,pp_avg_latency_sec,tg_throughput_tps,tg_avg_latency_sec,pp+tg_throughput_tps,concurrency,start,finish
+10,8,1,512,256,3530.8806353706973,1.4502000000000002,182.24422574354912,0.054871484375,494.62227088297806,10,1730390309.5198257,1730390326.3122041
+8,10,1,512,256,3224.8401299851535,1.2706250000000001,182.03718657685553,0.043947265624999995,488.54961832061076,8,1730390326.5202794,1730390340.2207406
+6,12,1,512,256,3145.0756048522235,0.9768333333333333,200.7467320316628,0.029889322916666666,529.5942995058039,6,1730390340.5268426,1730390350.2036488
+5,16,1,512,256,3376.9013889415573,0.7582000000000001,189.85063658241233,0.026336718749999998,509.08126740023863,5,1730390350.5240867,1730390359.030866
+4,20,1,512,256,3093.3413687653324,0.66225,173.75911547567443,0.0230205078125,467.08225634787897,4,1730390359.5313227,1730390367.0040333
+3,24,1,512,256,2859.001053909882,0.5383333333333333,167.00596832446584,0.017963541666666666,445.389522520781,3,1730390367.5333128,1730390373.54813
+2,32,1,512,256,2416.7997479603255,0.4345,153.94179614612153,0.0129921875,402.72679601468275,2,1730390374.541417,1730390379.1347277
+2,40,1,512,256,2713.387241689128,0.3775,137.8428197104192,0.014509765625,373.99561723886046,2,1730390379.5326273,1730390384.429726
+1,48,1,512,256,1868.6131386861312,0.274,100.6685017695635,0.00993359375,272.6304579339723,1,1730390385.5337763,1730390389.053051
+1,64,1,512,256,2015.748031496063,0.254,98.6892829606785,0.0101328125,269.6629213483146,1,1730390389.539855,1730390393.0968847
+1,80,1,512,256,2039.8406374501992,0.251,97.59817003431185,0.01024609375,267.223382045929,1,1730390393.5408826,1730390397.1080835
+10,8,2,512,256,3277.3050785364285,3.1246,337.7445326918161,0.05921640625,838.2906729247394,20,1730390397.544244,1730390417.1250806
+8,10,2,512,256,3045.733830735233,2.690125,305.39354882452415,0.0523916015625,760.5372284458748,16,1730390417.5578084,1730390434.845988
+6,12,2,512,256,2930.182417319503,2.097166666666667,319.55082864451447,0.037552734375,785.0753897265524,12,1730390435.559679,1730390448.3070822
+5,16,2,512,256,2989.757644656092,1.7126000000000001,312.24121605779914,0.0320265625,773.647627682079,10,1730390448.5653267,1730390459.443556
+4,20,2,512,256,2832.843355898869,1.446,311.6845406033694,0.025666992187499998,764.4643523702873,8,1730390459.5685928,1730390468.526116
+3,24,2,512,256,2659.9197311656208,1.1566666666666665,284.3603859211542,0.021100260416666666,698.6052152819891,6,1730390469.5763528,1730390477.0371006
+2,32,2,512,256,2517.563503936413,0.8135,227.05117638648886,0.0176171875,576.2521102982554,4,1730390477.5742445,1730390483.6777573
+2,40,2,512,256,2390.617054314159,0.8585,249.9111935656254,0.016005859375,616.6198313930149,4,1730390484.5803785,1730390490.3578148
+1,48,2,512,256,1762.4784853700517,0.581,169.76127320954907,0.01178125,427.02251876563804,2,1730390490.5788095,1730390494.8569124
+1,64,2,512,256,1906.8901303538173,0.537,170.2127659574468,0.01175,433.28631875881524,2,1730390495.5817165,1730390499.8454425
+1,80,2,512,256,1861.8181818181818,0.55,168.75411997363219,0.0118515625,428.5714285714286,2,1730390500.5866737,1730390504.874472
+10,8,4,512,256,2693.0077387282513,7.6049999999999995,422.7401393238641,0.09462109375,963.0999780543625,40,1730390505.5959723,1730390538.7304692
+8,10,4,512,256,2569.2506016754123,6.3774999999999995,420.2435014032347,0.076146484375,945.2671256586791,32,1730390539.60972,1730390566.7161942
+6,12,4,512,256,2398.7729152153534,5.1226666666666665,415.2524235916849,0.057796875,923.3081200220407,24,1730390567.606199,1730390588.5737839
+5,16,4,512,256,2572.8289928595045,3.9802,414.18744462464645,0.048287500000000004,939.047502598276,20,1730390589.6083093,1730390606.9843187
+4,20,4,512,256,2311.823224823339,3.5437499999999997,390.5044390871328,0.04097265625,873.9065500320035,16,1730390607.616382,1730390622.5760655
+3,24,4,512,256,2236.0819255315505,2.747666666666666,370.9556990049171,0.03234895833333334,834.3291689299293,12,1730390623.6240108,1730390635.5047665
+2,32,4,512,256,2068.691617749352,1.98,319.8501014856554,0.025011718750000002,732.4749642346209,8,1730390635.632079,1730390644.793665
+2,40,4,512,256,2071.6466284180287,1.9775,335.21663789306507,0.023865234375000002,758.3312762280918,8,1730390645.6316109,1730390654.5435154
+1,48,4,512,256,1478.7003610108304,1.385,230.2158273381295,0.017375,526.6586662094977,4,1730390655.6358361,1730390662.1596642
+1,64,4,512,256,1651.6129032258066,1.24,230.4749043439118,0.01735546875,540.559563610769,4,1730390662.639001,1730390669.0285537
+1,80,4,512,256,1612.5984251968503,1.27,234.59335624284077,0.01705078125,545.1641526175688,4,1730390669.6416163,1730390675.9844398
+10,8,8,512,256,1970.340709797918,20.788500000000003,603.2994576978308,0.13261367187499998,1116.603663855772,80,1730390676.6540015,1730390732.9443917
+8,10,8,512,256,1941.6876230033363,16.8765,589.1189310443326,0.10864208984374998,1091.9754732071447,64,1730390733.6598516,1730390779.8031938
+6,12,8,512,256,1836.1035701270516,13.385,596.193230501607,0.08051106770833334,1082.2935321922432,48,1730390780.6716452,1730390815.7982898
+5,16,8,512,256,1818.8591683173402,11.26,608.1152464706483,0.06577734375,1087.7031476826116,40,1730390816.678214,1730390845.8411481
+4,20,8,512,256,1747.1760984570924,9.377500000000001,551.5596446145637,0.058017578125000004,1010.8588351431391,32,1730390846.6866858,1730390871.8780494
+3,24,8,512,256,1579.2381871771217,7.780999999999999,471.79326154120827,0.050869791666666664,885.6428983278879,24,1730390872.6924965,1730390894.3495739
+2,32,8,512,256,1497.5243615719965,5.4704999999999995,391.101282762991,0.040910156249999996,768.9130842875915,16,1730390894.7021048,1730390911.472005
+2,40,8,512,256,1569.299959181721,5.2225,397.11243199334814,0.040291015625,784.2736788358437,16,1730390911.700744,1730390928.1677656
+1,48,8,512,256,1096.9469737546867,3.734,230.18995166910193,0.03475390625,486.42229435515793,8,1730390928.7004366,1730390942.0376577
+1,64,8,512,256,1245.363332319854,3.289,244.5957243520841,0.03270703125,526.8393071514321,8,1730390942.7080522,1730390955.0815513
+1,80,8,512,256,1221.2283840190817,3.354,248.90617403986388,0.032140625,530.4783284406838,8,1730390955.7080708,1730390967.9802835
+10,8,16,512,256,1268.6260034316892,64.57469999999999,411.97036952002105,0.388398046875,743.5781065625852,160,1730390968.7227573,1730391135.2166855
+8,10,16,512,256,1222.9198400165415,53.591625,419.91522662748946,0.3048388671875,742.2474913357646,128,1730391135.766095,1730391269.3404877
+6,12,16,512,256,1187.7725289570944,41.38216666666667,466.04694134007326,0.20598828125000002,782.4010696889625,96,1730391269.794422,1730391365.0658224
+5,16,16,512,256,1178.9692566103583,34.7428,484.88464508468036,0.16498828125,796.4223216021776,80,1730391365.8075807,1730391443.969276
+4,20,16,512,256,1124.0851868829982,29.1525,417.1361143601948,0.15343457031250002,713.2781889420984,64,1730391444.8251755,1730391514.6793184
+3,24,16,512,256,1083.080222509007,22.691,402.9691701445869,0.11911588541666666,692.3987152758212,48,1730391514.842891,1730391568.9518669
+2,32,16,512,256,948.7136525129333,17.27,331.0895905363526,0.09665429687499999,583.822306687255,32,1730391569.8458312,1730391612.6663566
+2,40,16,512,256,1008.1054978827249,16.252499999999998,375.7197650689609,0.085169921875,644.4304594084331,32,1730391612.8514872,1730391651.7814398
+1,48,16,512,256,715.7085444696837,11.446,230.34529299291418,0.0694609375,420.4187765156699,16,1730391652.8622456,1730391682.773916
+1,64,16,512,256,801.5655577299412,10.22,253.23029366306028,0.06318359375,465.542716423565,16,1730391683.8713672,1730391710.954842
+1,80,16,512,256,823.3993366167454,9.949,265.52573577077663,0.0602578125,484.25615763546796,16,1730391711.8685548,1730391737.9332025
+10,8,32,512,256,723.5379920854346,226.44940000000003,230.91732677824857,1.3858328125,419.15105538197054,320,1730391738.8811188,1730392326.446256
+8,10,32,512,256,682.9340768317518,191.9325,231.58949972946843,1.1054204101562501,412.075025622646,256,1730392327.0181077,1730392805.2551062
+6,12,32,512,256,688.7523020827139,142.72833333333332,249.47435632739473,0.7696608072916667,431.76514337416074,192,1730392806.1260765,1730393148.662534
+5,16,32,512,256,668.7214695562104,122.51100000000001,231.89792916415877,0.689975,408.1944504423103,160,1730393149.1671705,1730393451.1411726
+4,20,32,512,256,648.6930569133116,101.03825,245.77723383148478,0.5208408203124999,415.4684924559402,128,1730393452.2387886,1730393689.730125
+3,24,32,512,256,625.6611777396606,78.56033333333333,238.83078013764055,0.4019700520833333,405.11003049534327,96,1730393690.2861643,1730393873.1015184
+2,32,32,512,256,544.3292502014792,60.199,214.28200422508127,0.29867187500000003,359.49796670664995,64,1730393873.3096135,1730394010.8107498
+2,40,32,512,256,599.1989199268719,54.686499999999995,238.8756708704056,0.267921875,398.3079690766763,64,1730394011.32163,1730394135.52247
+1,48,32,512,256,421.09591857715634,38.908,161.35195288649032,0.19832421875,274.0440905897702,32,1730394136.3542788,1730394226.7150605
+1,64,32,512,256,465.8913185657008,35.167,178.84900882019036,0.178921875,303.5160736560003,32,1730394227.363666,1730394309.0610814
+1,80,32,512,256,492.72224227114157,33.252,194.47345931060676,0.164546875,326.04542559966035,32,1730394309.3754365,1730394385.4814527