Merge branch 'gcarreno:main' into main

Author: georges-hatem

entries/bfire/src/MultiThreadUnit.pas

@@ -0,0 +1,62 @@
+# Brian Fire
+
+An Entry to the One Billion Row Challenge in Object Pascal using Delphi 12 by [EagleAglow](https://github.com/EagleAglow), Discord: briar.on.fire
+
+## Compiler
+
+**Delphi 12** Professional Edition
+
+### Dependencies
+
+Project uses Delphi System units: `Classes`, `SysUtils`, `StrUtils`, `Diagnostics`,
+`Threading` and `SyncObjs`.
+
+### Execution
+```
+    Usage
+    bfire -h                       |  Write this help message and exit
+    bfire -v                       |  Write the version and exit
+    bfire -i <file_1>              |  <file_1> contains Weather Data
+    bfire -i <file_1> -o <file_2>  |  <file_1> contains Weather Data
+                                   |  <file_2> contains result
+    If <file_2> is not defined, result goes to CONSOLE (STDOUT)
+```
+
+#### Contest Mode
+
+To run the challenge, read from the 'challenge.csv' file:
+
+```console
+C:> bfire -i challenge.csv
+```
+
+## Remarks
+
+I haven't used Delphi very much recently, really needed to work on this for a refresher.
+I like TStringList self-sorting, but it is not as fast as other techniques.
+Now that this entry is set up, I can play with improvements. Maybe even get a time under 15 minutes! :)
+
+Second version uses hash of station name to accumulate data and fill a TStringList for Unicode station name.
+The list is initially unsorted and has linked objects for records holding accumulated data for each station.
+Finally, the TStringList is sorted and used to output sorted data.
+
+Third version has a thread for the console (which waits for tabulation, then sorts and writes results),
+two threads to read file, five threads to tabulate stations.  Stations are grouped into five separate stacks,
+so each tabulation thread has roughly the same work load. File is read byte-wise into "classic" byte array
+for each file line ending in ascii 10.  Each of these arrays is queued as a record in a last-in-first-out stack.
+Tabulation threads split the data into station name and temperature, then hash station name and use hash
+as index into one of five data arrays.  After all data is read and tabulated, the five data arrays are added to an
+initially unsorted TStringList that holds unsorted Unicode station name and has linked pointers to
+tabulated data for each station.  Finally, the TStringList is sorted, and the data is output.
+
+## History
+
+- Version 1.0: first working version, based on TStringList.
+- Version 1.1: modified rounding to new baseline.
+- Version 2.0: use hashing, sort later.
+- Version 2.1: minor speed tweaks.
+- Version 2.2: try hash functions modification.
+- Version 3.0: Six threads: one to read, four to tabulate, one (console) to rule them all...
+- Version 3.1: Safer locking strategy - didn't work.
+- Version 3.2: Eigth threads: two to read, five to tabulate, one (console) to rule them all...
+

entries/bfire/src/ProcessByHashUnit.pas

@@ -13,7 +13,6 @@ uses
   ConsoleUnit in 'ConsoleUnit.pas';
 
 var
-  UseStdOut: Boolean; // True unless output file is defined
   start: TDateTime; // for timing
 
 begin
@@ -49,14 +48,16 @@ begin
 
       if Not(UseStdOut) then // wait and report
       begin
-        while Not(ReadFile_Done and ParseDataQ_Done1 and ParseDataQ_Done2 and
-          ParseDataQ_Done3 and ParseDataQ_Done4) do
+        // while Not(ReadFile_Done1 and ReadFile_Done2 and ParseDataQ_Done1 and
+        // ParseDataQ_Done2 and ParseDataQ_Done3 and ParseDataQ_Done4 and
+        // ParseDataQ_Done5 and ParseDataQ_Done6) do
+        while Not(ReadFile_Done1 and ReadFile_Done2 and ParseData_Done) do
         begin
           Sleep(1000);
           WriteLn('Lines: ' + IntToStr(LineCount) + '  Stacks: ' +
             IntToStr(DataStackCount1) + ' / ' + IntToStr(DataStackCount2) +
             ' / ' + IntToStr(DataStackCount3) + ' / ' +
-            IntToStr(DataStackCount4));
+            IntToStr(DataStackCount4) + ' / ' + IntToStr(DataStackCount5));
 
           if DataStackCount1 > StackMax1 then
             StackMax1 := DataStackCount1;
@@ -66,28 +67,30 @@ begin
             StackMax3 := DataStackCount3;
           if DataStackCount4 > StackMax4 then
             StackMax4 := DataStackCount4;
-
-          if ReadFile_Done then
-            WriteLn('Done reading file');
-          if ParseDataQ_Done1 then
-            WriteLn('Done with ParseDataQ1');
-          if ParseDataQ_Done2 then
-            WriteLn('Done with ParseDataQ2');
-          if ParseDataQ_Done3 then
-            WriteLn('Done with ParseDataQ3');
-          if ParseDataQ_Done4 then
-            WriteLn('Done with ParseDataQ4');
+          if DataStackCount5 > StackMax5 then
+            StackMax5 := DataStackCount5;
+
+          if ReadFile_Done1 then
+            WriteLn('Done with reading thread 1');
+          if ReadFile_Done2 then
+            WriteLn('Done with reading thread 2');
+          if ParseData_Done then
+            WriteLn('Done with tabulating threads');
         end;
       end
       else // just wait
       begin
-        while Not(ReadFile_Done and ParseDataQ_Done1 and ParseDataQ_Done2 and
-          ParseDataQ_Done3 and ParseDataQ_Done4) do
+        // while Not(ReadFile_Done1 and ReadFile_Done2 and ParseDataQ_Done2 and
+        // ParseDataQ_Done2 and ParseDataQ_Done3 and ParseDataQ_Done4 and
+        // ParseDataQ_Done5 and ParseDataQ_Done6) do
+        while Not(ReadFile_Done1 and ReadFile_Done2 and ParseData_Done) do
         begin
           Sleep(100);
         end;
       end;
 
+      Challenge.Free;
+
       SortArrays; // sort
       ArrayToFile(outputFilename, UseStdOut); // output
 
@@ -97,7 +100,8 @@ begin
           Now - start)]));
 
         WriteLn('Stack Max: ' + IntToStr(StackMax1) + '/' + IntToStr(StackMax2)
-          + '/' + IntToStr(StackMax3) + '/' + IntToStr(StackMax4));
+          + '/' + IntToStr(StackMax3) + '/' + IntToStr(StackMax4) + '/' +
+          IntToStr(StackMax5));
 
         WriteLn('Press ENTER to exit');
         readln;

entries/bfire/src/README.md

@@ -1,2 +1,2 @@
 const
-  cVersion = '3.1';
+  cVersion = '3.2';

entries/bfire/src/bfire.dpr

@@ -3,42 +3,33 @@
 **1 billion row Challenge entry**
 
 ## Version
-Version 2.00 (first version with threads)
+Version 2.10 (version with threads, does NOT yet work for 400 stations or 5 billion rows)
 
 ## How to compile
 The program was developed with FPC 3.2.2 and Lazarus 2.2.4
 
-1brc.lpi + 1brc.pas = version without threads
+It uses package **mORMot 2** to compile, which you can download from https://github.com/synopse/mORMot2
 
-1brc_th.lpi + 1brc_th.pas = version with threads
+Files to compile: **1brc_th.lpi** and **1brc_th.pas** = this version with threads
 
 ## How to start
 ```
-Usage: <path to input file> <thread count> [<bit-width for hash-list> [buffer size in KB]]
- - thread count: allowed range = [1..32]
+Usage: <path to input file> <thread count> [bit-width for hash-list]
+ - thread count: allowed range = [1..64]
  - bit-width for hash-list: sets the size of the hash list, e.g. '16' => 65536 entries,
-   allowed range = [16..28], Default=18
- - buffer size in KB: allowed range = [1..2,000,000 KB], Default=128 KB
-Example: hgrosser measurements.txt 32 18 128
+   allowed range = [16..28], Default=16
+Example: hgrosser measurements.txt 32 16
 ```
-There are no switches like `-i` etc, only 2..4 values.
+There are no switches like `-i` etc, only 2 or 3 values.
 
-### Optimizing the 2nd command line parameter
-This parameter sets the thread count. With my own old CPU I could only test 1..4 threads. Although I expect 32 threads to be the fastest, I would be very interested to see the results for 1, 4, 8, 16 and 32 threads. Please use for this test the defaults for the 3rd and 4th command line parameters.
-
-### Optimizing the 3rd command line parameter
-In theory the program should run faster with greater bit-widths for the hash-list (because of less collisions). On the computer of Gus - without threads - 18 bits was the fastest. Please try the values from 16 to 20 again and use '32' for the 2nd command line parameter and the default for the 4th command line parameter.
-
-### Optimizing the 4th command line parameter
-After the 2nd and 3rd command line parameters had been optimized, please try with them 64, 96, 128, 192 and 256 KB as the 4th command line parameter. Thanks a lot!
+There is no more need for further optimizing. Please use '32' as 2nd command line parameter and '16' as 3rd command line parameter.
 
 ## How the program works
-The Program works with multi threads.
+The Program works with threads.
 
 To speed things up:
 
-- the input file is read via procedure 'blockread' ...
-- into an AnsiString, so that function 'PosEx' can be used to parse it
+- the input file is read via **mORMot**'s type 'TMemoryMap'
 - to manage the city names, a self made hash-list is used
 - temperatures are stored as integers (multiplied by 10)
 
@@ -50,4 +41,4 @@ To speed things up:
 - Version 1.60: hash-list optimized, some minor improvements, Conditional "noCR" added
 - Version 1.61: Conditional "noCR" constantely enabled => input files must NOT have CR's
 - Version 2.00: 1st version with threads
-
+- Version 2.10: the input file is now read via **mORMot**'s type 'TMemoryMap', the hash code is now computed via **mORMot**'s crc32c-function, several small improvements

entries/bfire/src/version.inc

@@ -11,7 +11,7 @@
         <CompatibilityMode Value="True"/>
       </Flags>
       <SessionStorage Value="InIDEConfig"/>
-      <Title Value="1brc"/>
+      <Title Value="1brc_old"/>
       <UseAppBundle Value="False"/>
       <ResourceType Value="res"/>
     </General>
@@ -24,7 +24,7 @@
         <CompilerOptions>
           <Version Value="11"/>
           <Target>
-            <Filename Value="../../../bin/hgrosser"/>
+            <Filename Value="../../../bin/hgrosser_old"/>
           </Target>
           <SearchPaths>
             <IncludeFiles Value="$(ProjOutDir)"/>
@@ -59,7 +59,7 @@
         <CompilerOptions>
           <Version Value="11"/>
           <Target>
-            <Filename Value="../../../bin/hgrosser"/>
+            <Filename Value="../../../bin/hgrosser_old"/>
           </Target>
           <SearchPaths>
             <IncludeFiles Value="$(ProjOutDir)"/>
@@ -104,15 +104,15 @@
     </RunParams>
     <Units Count="1">
       <Unit0>
-        <Filename Value="1brc.pas"/>
+        <Filename Value="1brc_old.pas"/>
         <IsPartOfProject Value="True"/>
       </Unit0>
     </Units>
   </ProjectOptions>
   <CompilerOptions>
     <Version Value="11"/>
     <Target>
-      <Filename Value="../../../bin/hgrosser"/>
+      <Filename Value="../../../bin/hgrosser_old"/>
     </Target>
     <SearchPaths>
       <IncludeFiles Value="$(ProjOutDir)"/>

entries/hgrosser/README.md

@@ -101,6 +101,11 @@
         </Mode0>
       </Modes>
     </RunParams>
+    <RequiredPackages Count="1">
+      <Item1>
+        <PackageName Value="mormot2"/>
+      </Item1>
+    </RequiredPackages>
     <Units Count="1">
       <Unit0>
         <Filename Value="1brc_th.pas"/>
@@ -115,6 +120,7 @@
     </Target>
     <SearchPaths>
       <IncludeFiles Value="$(ProjOutDir)"/>
+      <OtherUnitFiles Value="../../../baseline/Common"/>
       <UnitOutputDirectory Value="../../../bin/lib/$(TargetCPU)-$(TargetOS)"/>
     </SearchPaths>
     <Parsing>