Corrections to version sync

Author: Chris Maunder

docs/WSL-README.md

@@ -0,0 +1,135 @@
+# Notes on developing under WSL
+
+This repository can be run under Windows and WSL at the same time. The setup.bat
+and setup.sh scripts will install runtimes and virtual environments into specific
+locations for the given OS. Virtual environments, for example, will live under
+the /bin/windows folder for a module in Windows, and /bin/linux when under Linux.
+This allows the same code and models to be worked on and tested under both 
+Operating Systems.
+
+Note that this means you will need VS Code installed in Windows and in the WSL
+hosted Ubuntu (should you choose to use Ubuntu in WSL). This further requires
+that each instance of VS Code has the necessary extensions installed. The
+profile sync mechanism in VS Code makes this seamless.
+
+## VSCode integrated terminals won't load .bashrc, and so $PATH may not be accurate
+
+If you are using VSCode in WSL, the integrated terminal doesn't load the login 
+stuff. This means your .bashrc isn't loaded.
+
+If you've installed a CUDA toolkit which has set paths in the `PATH` variable 
+and updated the .bashrc file, the `PATH` variable won't have the CUDA paths in 
+it, meaning you won't get access to all the CUDA tools (typically just 
+`nvidia-smi`, not `nvcc`).
+
+If the install scripts can only see `nvidia-smi` and not `nvcc` then those 
+scripts will potentially see an incorrect version of CUDA. `nvidia-smi` reports
+the maximum CUDA version the drivers can support, whereas `nvcc` reports the 
+actual CUDA version installed and accessible.
+
+Getting the wrong version of CUDA means potentially the wrong version of Python 
+libraries such as PyTorch will be installed, leading to instability or failure.
+
+Install CodeProject.AI on a CUDA under WSL using a terminal window, not the 
+integrated VSCode terminal
+
+## Getting a VSCode Integrated terminal to load login profiles under WSL
+
+To get the login profile loaded add this to your settings.json file
+
+```json
+"terminal.integrated.shellArgs.windows": ["-l"],
+"terminal.integrated.shellArgs.linux": ["-l"],
+"terminal.integrated.shellArgs.osx": ["-l"],
+```
+The settings files are typically located in
+
+| OS      | settings location                                             |
+|---------|---------------------------------------------------------------|
+| Windows | `C:\Users\<username>\AppData\Roaming\Code\User\settings.json` |
+| Linux   | `$HOME/.config/Code/User/settings.json`                       |
+| macOS   | `$HOME/Library/Application\ Support/Code/User/settings.json`  |
+
+You can also set your default profiles to achieve the same result (replace `osx`
+with `Windows` and `Linux` for other OS's) 
+
+```json
+"terminal.integrated.profiles.osx": {
+  "bash": {
+    "path": "bash",
+    "args": ["-l"]
+  }
+} 
+```
+
+See this [Stackoverflow discussion](https://stackoverflow.com/questions/51820921/vscode-integrated-terminal-doesnt-load-bashrc-or-bash-profile) for more information.
+
+## Speed considerations
+
+To share the same files and code between WSL (Ubuntu) and Windows, you need to
+have one environment point to files in another. Often this would mean that the
+file system lives in Windows, and the WSL instance accesses the Windows hosted
+files through the magic of WSL. 
+
+Crossing the OS boundary will result in poor disk performance for the WSL
+instance. Having a WSL instance of VS Code work on its own copy of this repo,
+separate from the Windows instance, speeds disk access (eg PIP installs)
+dramatically.
+
+If you want speed, have a separate installation of this project in WSL.
+
+## Space considerations
+
+If you choose to run separate copies of the code in WSL and Windows then it means
+you are doubling up on the code, libraries, tools, and compiled executables. If
+you have limited disk space this can be an issue.
+
+If you are low on space, use a shared installation of this project for WSL and
+Windows.
+
+### To free up space
+
+To free up space you can use the clean.bat/clean.sh scripts under
+/src/SDK/Utilities.
+
+For Windows
+```cmd 
+cd devops\install
+clean all
+```
+For Linux/macOS
+```cmd 
+cd devops/install
+bash clean.sh all
+```
+
+Run `clean` without a parameter to see the options for fine-tuning what gets
+cleaned.
+
+To actually realise the freed up space in WSL you will need to compact the VHD
+in which your WSL instance resides.
+
+In a Windows terminal:
+
+```cmd 
+wsl --shutdown
+diskpart
+```
+
+This will shutdown WSL and open a disk partition session in a new window. Locate
+the VHD file for WSL by heading to `%LOCALAPPDATA%\Packages` and looking for a 
+folder similar to `CanonicalGroupLimited.Ubuntu_79rhkp1fndgsc\LocalState` that
+contains a file `ext4.vhd`.
+
+Within the session, enter the following (adjusting the `ext4.vhd` location as needed)
+
+```text
+select vdisk file="%LOCALAPPDATA%\Packages\CanonicalGroupLimited.Ubuntu_79rhkp1fndgsc\LocalState\ext4.vhdx"
+attach vdisk readonly
+compact vdisk
+detach vdisk
+exit
+```
+
+Your WSL virtual hard drive should be smaller and the space that was used 
+reclaimed by Windows.

modules/ObjectDetectionYOLOv5-6.2/modulesettings.radxarock.json

@@ -0,0 +1,15 @@
+{
+  "Modules": {
+
+    "ObjectDetectionYOLOv5-6.2": {
+      "LaunchSettings": {
+        "AutoStart": false,
+        "Runtime": "python3.8"
+      },
+      "EnvironmentVariables": {
+        "MODEL_SIZE": "Tiny", // tiny, small, medium, large
+        "USE_CUDA": "False"
+      }
+    }
+  }
+}

src/SDK/NET/Backend/ModuleWorkerBase.cs

@@ -393,68 +393,6 @@ private ExpandoObject ProcessModuleCommands(BackendRequest request)
                     }.ToExpando();
                 }
             }
-            else
-                response = moduleResponse.ToExpando();
-            return response;
-        }
-
-        /// <summary>
-        /// Processes a command sent to a module.
-        /// </summary>
-        /// <param name="request">The incoming request</param>
-        /// <returns>An expando object with the results</returns>
-        private ExpandoObject ProcessModuleCommands(BackendRequest request)
-        {
-            ExpandoObject response;
-
-            ModuleResponse moduleResponse = Process(request);
-
-            if (!_doNotLogCommands.Contains(request.reqtype))
-                UpdateStatistics(moduleResponse);
-
-            if (moduleResponse.LongProcessMethod is not null)
-            {
-                if (_longRunningTask is not null && !_longRunningTask.IsCompleted)
-                {
-                    response = new
-                    {
-                        Success   = false,
-                        CommandId = _longRunningCommandId,
-                        Error     = "A long running command is already in progress"
-                    }.ToExpando();
-                }
-                else
-                {
-                    // We have a previous long running process that is now done, but we
-                    // have not stored (nor returned) the result. We can read the result
-                    // now, but we have to start a new process, so...???
-                    // if (_longRunningTask is not null && _longRunningTask.IsCompleted &&
-                    //     _lastLongRunningLastOutput is null)
-                    //    _lastLongRunningLastOutput = ...
-
-                    // Store request Id as the command Id for later, reset the last result
-                    string? commandId = request.reqid;
-
-                    _longRunningCommandId  = commandId;
-                    _lastLongRunningOutput = null;
-                    _longRunningTask       = null;
-
-                    // Start the long running process
-                    Console.WriteLine("Starting long process with command ID " + commandId);
-
-                    _longProcessCancellationTokenSource = new CancellationTokenSource();
-                    CancellationToken cancellationToken = _longProcessCancellationTokenSource.Token;
-                    _longRunningTask = moduleResponse.LongProcessMethod(request, cancellationToken);
-
-                    response = new
-                    {
-                        Success = true,
-                        CommandId = commandId,
-                        Message = "Command is running in the background",
-                        CommandStatus = "running"
-                    }.ToExpando();
-                }
-            }
             else
             {
                 response = moduleResponse.ToExpando();

src/SDK/Python/Python.pyproj

@@ -18,6 +18,8 @@
     <InterpreterId>MSBuild|env|$(MSBuildProjectFullPath)</InterpreterId>
     <IsWindowsApplication>False</IsWindowsApplication>
     <EnableNativeCodeDebugging>False</EnableNativeCodeDebugging>
+    <SuppressPackageInstallationPrompt>True</SuppressPackageInstallationPrompt>
+    <SuppressEnvironmentCreationPrompt>True</SuppressEnvironmentCreationPrompt>
   </PropertyGroup>
 
   <ItemGroup>

src/server/Program.cs

@@ -92,8 +92,7 @@ public static async Task Main(string[] args)
 
             // GetProcessStatus a directory for the given platform that allows modules to store persisted data
             string programDataDir     = Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData);
-            string applicationDataDir = programDataDir + "/" + companyName + "/" + productCat;
-            applicationDataDir = applicationDataDir.Replace('/', Path.DirectorySeparatorChar);
+            string applicationDataDir = $"{programDataDir}\\{companyName}\\{productCat}".Replace('\\', Path.DirectorySeparatorChar);
 
             // .NET's suggestion for macOS and Linux aren't great. Let's do something different.
             if (SystemInfo.IsMacOS)

src/server/appsettings.development.json

@@ -28,7 +28,7 @@
     "ModulesDirPath":             "%ROOT_PATH%/modules",
     "PreInstalledModulesDirPath": "%ROOT_PATH%/preinstalled-modules",
     "DemoModulesDirPath":         "%ROOT_PATH%/src/demos/modules",
-    "ExternalModulesDirPath":     "",
+    "ExternalModulesDirPath":     "%ROOT_PATH%/../CodeProject.AI-Modules", // ** %EXTERNAL_MODULES_PATH% will expand to this value
 
     "DownloadedModulePackagesDirPath": "%ROOT_PATH%/downloads/modules",
     "DownloadedModelsPackagesDirPath": "%ROOT_PATH%/downloads/models",

src/server/appsettings.json

@@ -95,6 +95,7 @@
     "ModuleListUrl":   "https://www.codeproject.com/ai/modules/list",  // Location of the JSON list of modules that can be downloaded
     "ModelListUrl":    "https://www.codeproject.com/ai/models/list",   // Location of the JSON list of models that can be downloaded
     // "AssetStorageUrl": "https://codeproject-ai.s3.ca-central-1.amazonaws.com/server/assets/",
+    // "AssetStorageUrl": "https://codeproject-ai-bunny.b-cdn.net/server/assets/",
     "AssetStorageUrl": "https://www.codeproject.com/ai/download/server/assets/", // Location of downloadable models
 
     // The location of the AI modules (pre-installed and downloaded/side-loaded)
@@ -107,7 +108,7 @@
     "ModulesDirPath":                  "%ROOT_PATH%/modules",                    // ** %MODULES_PATH% will expand to this value
     "PreInstalledModulesDirPath":      "%ROOT_PATH%/preinstalled-modules",       // ** %PREINSTALLED_MODULES_PATH% will expand to this value
     "DemoModulesDirPath":              "%ROOT_PATH%/src/demos/modules",          // ** %DEMO_MODULES_PATH% will expand to this value
-    "ExternalModulesDirPath":          "",                                       // ** %EXTERNAL_MODULES_PATH% will expand to this value
+    "ExternalModulesDirPath":          "%ROOT_PATH%/../CodeProject.AI-Modules",  // ** %EXTERNAL_MODULES_PATH% will expand to this value
 
     // Downloads for modules and models go here
     "DownloadedModulePackagesDirPath": "%ROOT_PATH%/downloads/modules",    // Where module packages are downloaded to

src/setup.bat

@@ -120,7 +120,9 @@ set appRootDirPath=!setupScriptDirPath!
 
 :: The location of large packages that need to be downloaded (eg an AWS S3 bucket
 :: name). This will be overwritten using the value from appsettings.json
-set assetStorageUrl=https://codeproject-ai.s3.ca-central-1.amazonaws.com/server/assets/
+REM set assetStorageUrl=https://codeproject-ai.s3.ca-central-1.amazonaws.com/server/assets/
+REM set assetStorageUrl=https://codeproject-ai-bunny.b-cdn.net/server/assets/
+set assetStorageUrl=https://www.codeproject.com/ai/download/server/assets/
 
 :: The name of the source directory (in development)
 set srcDirName=src

src/setup.sh

@@ -137,7 +137,9 @@ appRootDirPath="${setupScriptDirPath}"
 
 # The location of large packages that need to be downloaded (eg an AWS S3 bucket
 # name). This will be overwritten using the value from appsettings.json
-assetStorageUrl='https://codeproject-ai.s3.ca-central-1.amazonaws.com/server/assets/'
+# assetStorageUrl='https://codeproject-ai.s3.ca-central-1.amazonaws.com/server/assets/'
+# assetStorageUrl='https://codeproject-ai-bunny.b-cdn.net/server/assets/'
+assetStorageUrl='https://www.codeproject.com/ai/download/server/assets/'
 
 # The name of the source directory (in development)
 srcDirName='src'