Merge pull request #37 from ikelaiah/ikelaiah-rev03

Update - Rev 3, 3-4 mins faster.

Author: gcarreno

entries/ikelaiah/README.md

@@ -6,7 +6,7 @@ The approach I implemented here is simplistic.
 
 - Sequentially read the measurement file.
 - Populate a `TDictionary` with station names, min, max, count and sum; without storing all the temperature measurements.
-- Avoided [StrUtil.SplitString](https://www.freepascal.org/docs-html/rtl/strutils/splitstring.html), used [`Pos()`](https://www.freepascal.org/docs-html/rtl/system/pos.html) and [`Copy()`](https://www.freepascal.org/docs-html/rtl/system/copy.html) instead.
+- Format output and sort in `TStringList`.
 - Use a custom comparer to sort the station and temperature statistics in a `TStringList`.
 - Use the rounding method as provided in the `baseline.lpr` (or the `README.md` from 1brc-ObjectPascal).
 - Display the sorted measurements using a simple `for` loop.
@@ -34,6 +34,30 @@ To time the execution, do the following.
 $ time ./OneBRC.exe -i your_measurements.txt
 ```
 
+### Save an output
+
+```bash
+$ ./ikelaiah.exe -i measurements.txt > ikelaiah-output.txt
+```
+
+### Verifying SHA256 output on Windows
+
+Launch `git bash` (make sure you have Git for Windows installed).
+
+Run dos2unix on your output. The `\r\n` on Windows changes calculation of `sha256`.
+
+```bash
+$ dos2unix.exe ikelaiah-output.txt
+dos2unix: converting file ikelaiah-output.txt to Unix format...
+```
+
+Run `sha256sum` on your output.
+
+```bash
+$ sha256sum.exe ikelaiah-output.txt
+db3d79d31b50daa8c03a1e4f2025029cb137f9971aa04129d8bca004795ae524 *ikelaiah-output.txt
+```
+
 ## Help
 
 To see flags, use `-h`.
@@ -54,7 +78,7 @@ Use `-v` to check version.
 
 ```bash
 $ ./OneBRC.exe -v
-OneBRC version 1.0
+OneBRC version 1.3
 ```
 
 ## Authors
@@ -65,7 +89,18 @@ Iwan Kelaiah
 ## Version History
 
 * 1.0
-    * Initial Release - Sequential approach.
+    * Initial Release - Sequential approach. 
+    * `AssignFile` -> `Reset` -> Parse weather station and the recorded temperature with `TStringHelper.Split` ->  `TDictionary` -> `TStringList` -> A `for` loop -> output.
+
+* 1.2
+    * Revision release - Sequential approach. 
+    * Replaced `TStringHelper.Split` with `Pos()` and `Copy()`. 2-3 mins faster for 1 billion rows.
+    * Float now stored as Int64. 2-3 mins faster for 1 billion rows.
+    * Applied baseline's rounding.
+
+* 1.3
+    * Revision release - Sequential approach.
+    * Replaced `AssignFile()` and `Reset()` with `TfileStream` and `TStreamReader`. 3-4 mins faster for 1 billion rows.
 
 ## License
 

entries/ikelaiah/src/OneBRC.lpr

@@ -32,7 +32,7 @@
   WeatherStation;
 
 const
-  version = '1.0';
+  version = '1.3';
 
 type
 
@@ -96,10 +96,16 @@   TOneBRC = class(TCustomApplication)
       Exit;
     end;
 
-    // Start the main algorithm
-    WeatherStation.ProcessTempMeasurements(filename);
+    // Start the main algorithm ////////////////////////////////////////////////
+    try
+      WeatherStation.ProcessTempMeasurementsV3(filename);
+    except
+      on E: Exception do
+        WriteLn('Error: ' + E.Message);
+    end;
+
 
-    // stop program loop
+    // Stop program loop ///////////////////////////////////////////////////////
     Terminate;
   end;
 

entries/ikelaiah/src/weatherstation.pas

@@ -7,12 +7,19 @@ interface
   Classes,
   SysUtils,
   Generics.Collections,
-  Math
+  Math,
+  streamex
   {$IFDEF DEBUG}
-,  Stopwatch
+  , Stopwatch
   {$ENDIF}
   ;
 
+type
+  TParsedData = record
+    wsName: string;
+    wsTemp: int64;
+  end;
+
 type
   // Create a record of temperature stats
   TStat = record
@@ -28,7 +35,6 @@   TStat = record
     constructor Create(const newMin: int64; const newMax: int64;
       const newSum: int64; const newCount: int64);
     function ToString: string;
-
   end;
 
 type
@@ -37,6 +43,7 @@   TStat = record
 
 // The main algorithm to process the temp measurements from various weather station
 procedure ProcessTempMeasurements(const filename: string);
+procedure ProcessTempMeasurementsV3(const filename: string);
 
 implementation
 
@@ -137,17 +144,26 @@ procedure AddCityTemperature(const cityName: string; const newTemp: int64;
 
     // Update the stat of this city
     weatherDictionary.AddOrSetValue(cityName, stat);
+    {$IFDEF DEBUG}
+    // Display the line.
+    // WriteLn('Updated: ', cityName);
+    {$ENDIF DEBUG}
   end;
 
   // If city name doesn't exist add a new entry
   if not weatherDictionary.ContainsKey(cityName) then
   begin
     weatherDictionary.Add(cityName, TStat.Create(newTemp, newTemp, newTemp, 1));
+    {$IFDEF DEBUG}
+    // Display the line.
+    // WriteLn('Added: ', cityName);
+    {$ENDIF DEBUG}
   end;
 end;
 
 procedure ProcessTempMeasurements(const filename: string);
 var
+  buf: array[0..2047] of char;
   wd: TWeatherDictionary;
   line, ws, strTemp: string;
   weatherStationList: TStringList;
@@ -164,6 +180,7 @@ procedure ProcessTempMeasurements(const filename: string);
 
     // Read text file //////////////////////////////////////////////////////////
     AssignFile(textFile, filename);
+    SetTextBuf(textFile, buf[0], sizeof(buf));
 
     // Perform the read operation in a try..except block to handle errors gracefully
     try
@@ -197,7 +214,10 @@ procedure ProcessTempMeasurements(const filename: string);
 
           // Add the weather station and the recorded temp (as int64) in the TDictionary
           Val(strTemp, intTemp, valCode);
-          if valCode <> 0 then Continue;
+
+          if valCode <> 0 then
+            Continue;
+
           AddCityTemperature(ws, intTemp, wd);
         end;
       end; // end while loop reading line at a time
@@ -258,4 +278,104 @@ procedure ProcessTempMeasurements(const filename: string);
 
 end;
 
+procedure ProcessTempMeasurementsV3(const filename: string);
+var
+
+  fReader: TStreamReader;
+  fStream: TFileStream;
+  line, parsedStation, strTemp, wsKey, outputList: string;
+  parsedTemp: int64;
+  delimiterPos, valCode, index: integer;
+  wd: TWeatherDictionary;
+  weatherStationList: TStringList;
+begin
+
+  // Create a city - weather dictionary
+  wd := TWeatherDictionary.Create;
+  // Create a city - weather TStringList for sorting
+  weatherStationList := TStringList.Create;
+
+  // Create a file stream, and use TStreamReader to read the contents
+  fStream := TFileStream.Create(filename, fmOpenRead);
+  try
+    fReader := TStreamReader.Create(fStream);
+    try
+      while not fReader.EOF do // Start read file
+      begin
+        line := fReader.ReadLine;
+        // Get position of the delimiter ---------------------------------------
+        delimiterPos := Pos(';', line);
+        if delimiterPos > 0 then
+        begin
+          // Get the weather station name
+          // Using Copy and POS - as suggested by Gemini AI.
+          // This part saves 3 mins faster when processing 1 billion rows.
+          parsedStation := Copy(line, 1, delimiterPos - 1);
+
+          // Get the temperature recorded, as string, remove '.' from string float
+          // because we want to save it as int64.
+          strTemp := Copy(line, delimiterPos + 1, Length(line));
+          strTemp := StringReplace(strTemp, '.', '', [rfReplaceAll]);
+
+          // Add the weather station and the recorded temp (as int64) in the TDictionary
+          Val(strTemp, parsedTemp, valCode);
+          if valCode <> 0 then
+            Continue;
+
+          // Add a record in TWeatherDictionary
+          AddCityTemperature(parsedStation, parsedTemp, wd);
+
+        end; // end of checking delimiter in a line ----------------------------
+      end; // end of reading file
+
+      {$IFDEF DEBUG}
+      Stopwatch.StopTimer;
+      WriteLn('Finished reading and parsing input file');
+      Stopwatch.DisplayTimer;
+      {$ENDIF}
+
+      // Format and sort weather station by name and temp stat /////////////////
+      {$IFDEF DEBUG}
+      Stopwatch.StartTimer;
+      {$ENDIF}
+      wsKey := '';
+      for wsKey in wd.Keys do
+      begin
+        weatherStationList.Add(wsKey + '=' + wd[wsKey].ToString + ', ');
+      end;
+      weatherStationList.CustomSort(@CustomTStringListComparer);
+
+      {$IFDEF DEBUG}
+      Stopwatch.StopTimer;
+      WriteLn('Finished creating TStringList and sorted it');
+      Stopwatch.DisplayTimer;
+      {$ENDIF}
+
+      // Print TStringList - sorted by weather station and temp stat ///////////
+      {$IFDEF DEBUG}
+      Stopwatch.StartTimer;
+      {$ENDIF}
+      outputList := '';
+      // Print the weather station and the temp stat
+      for index := 0 to weatherStationList.Count - 1 do
+        outputList := outputList + weatherStationList[index];
+      // Remove last comma and space; ', ', a neat trick from Gus.
+      SetLength(outputList, Length(outputList) - 2);
+      WriteLn('{', outputList, '}');
+
+      {$IFDEF DEBUG}
+      Stopwatch.StopTimer;
+      WriteLn('Finished printing the sorted weather station and temperatures');
+      Stopwatch.DisplayTimer;
+      {$ENDIF}
+    finally
+      fReader.Free;
+    end;
+  finally
+    fStream.Free;
+    weatherStationList.Free;
+    wd.Free;
+  end;
+end;
+
 end.