Skip to content

Commit a08f5a3

Browse files
author
Chris Maunder
committed
Module package downloads, package creation, and module asset downloads locations cleaned up
1 parent 2b05d5a commit a08f5a3

File tree

7 files changed

+81
-68
lines changed

7 files changed

+81
-68
lines changed

‎src/scripts/utils.bat‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -312,8 +312,8 @@ shift & goto :%~1
312312
REM - downloadDirPath is the path where downloads are stored (typically src/downloads)
313313
if /i "%forceOverwrite%" == "true" (
314314
REM Force Re-download, then force re-copy of downloads to install dir
315-
if exist "!downloadDirPath!\!modulesDir!\!moduleDirName!\!fileToGet!" (
316-
del /s %rmdirFlags% "!downloadDirPath!\!modulesDir!\!moduleDirName!\!fileToGet!"
315+
if exist "!downloadModuleAssetsDirPath!\!moduleDirName!\!fileToGet!" (
316+
del /s %rmdirFlags% "!downloadModuleAssetsDirPath!\!moduleDirName!\!fileToGet!"
317317
)
318318
if exist "!moduleDirPath!\!moduleAssetsDirName!" rmdir /s %rmdirFlags% "!moduleDirPath!\!moduleAssetsDirName!"
319319
)
@@ -322,28 +322,28 @@ shift & goto :%~1
322322

323323
REM Params are: S3 storage bucket | fileToGet | zip lives in... | zip expanded to moduleDir/... | message
324324
REM eg "S3_bucket/folder" "rembg-models.zip" \downloads\myModuleDir" "assets" "Downloading models..."
325-
call :DownloadAndExtract "!assetStorageUrl!!folder!" "!fileToGet!" "!downloadDirPath!\!modulesDir!\!moduleDirName!" "!moduleAssetsDirName!" "!message!"
325+
call :DownloadAndExtract "!assetStorageUrl!!folder!" "!fileToGet!" "!downloadModuleAssetsDirPath!\!moduleDirName!" "!moduleAssetsDirName!" "!message!"
326326

327327
REM Copy downloadDirPath\modules\moduleDirName\moduleAssetsDirName folder to modulesDirPath\moduleDirName\
328-
if exist "!downloadDirPath!\!modulesDir!\!moduleDirName!\!moduleAssetsDirName!" (
328+
if exist "!downloadModuleAssetsDirPath!\!moduleDirName!\!moduleAssetsDirName!" (
329329

330330
REM if /i "%verbosity%" neq "quiet" ( ... )
331331

332332
call :Write "Copying contents of !fileToGet! to !moduleAssetsDirName!..."
333333

334-
REM move "!downloadDirPath!\!modulesDir!\!moduleDirName!\!moduleAssetsDirName!" !moduleDirPath!
334+
REM move "!downloadModuleAssetsDirPath!\!moduleDirName!\!moduleAssetsDirName!" !moduleDirPath!
335335
REM if errorlevel 1 (
336336
REM call :WriteLine "Failed" !color_error!
337337
REM ) else (
338338
REM call :WriteLine "done" !color_success!
339339
REM )
340340

341-
robocopy /E "!downloadDirPath!\!modulesDir!\!moduleDirName!\!moduleAssetsDirName! " ^
341+
robocopy /E "!downloadModuleAssetsDirPath!\!moduleDirName!\!moduleAssetsDirName! " ^
342342
"!moduleDirPath!\!moduleAssetsDirName! " !roboCopyFlags! /MOVE >NUL
343343
if errorlevel 16 (
344344
call :WriteLine "Failed" !color_error!
345345
else if errorlevel 8 (
346-
call :WriteLine "Some files not copied" !color_warn!
346+
call :WriteLine "Some files inot copied" !color_warn!
347347
) else (
348348
call :WriteLine "done" !color_success!
349349
)

‎src/scripts/utils.sh‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2058,30 +2058,30 @@ function getFromServer () {
20582058
if [ "${forceOverwrite}" = true ]; then
20592059
# if [ $verbosity -ne "quiet" ]; then echo "Forcing overwrite"; fi
20602060

2061-
rm -rf "${downloadDirPath}/${modulesDir}/${moduleDirName}/${fileToGet}"
2061+
rm -rf "${downloadModuleAssetsDirPath}/${moduleDirName}/${fileToGet}"
20622062
rm -rf "${moduleDirPath}/${moduleAssetsDirName}"
20632063
fi
20642064

20652065
# Download !$assetStorageUrl$folder$fileToGet to $downloadDirPath and extract into $downloadDirPath/${modulesDir}/$moduleDirName/$moduleAssetsDirName
20662066
# Params are: S3 storage bucket | fileToGet | zip lives in... | zip expanded to moduleDir/... | message
20672067
# eg "S3_bucket/folder" "rembg-models.zip" /downloads/myModuleDir/" "assets" "Downloading models..."
2068-
downloadAndExtract "${assetStorageUrl}${folder}" "$fileToGet" "${downloadDirPath}/${modulesDir}/${moduleDirName}" "${moduleAssetsDirName}" "${message}"
2068+
downloadAndExtract "${assetStorageUrl}${folder}" "$fileToGet" "${downloadModuleAssetsDirPath}/${moduleDirName}" "${moduleAssetsDirName}" "${message}"
20692069

20702070
# Copy downloadDirPath/modules/moduleDirName/moduleAssetsDirName folder to modules/moduleDirName/moduleAssetsDirName
2071-
if [ -d "${downloadDirPath}/${modulesDir}/${moduleDirName}/${moduleAssetsDirName}" ]; then
2071+
if [ -d "${downloadModuleAssetsDirPath}/${moduleDirName}/${moduleAssetsDirName}" ]; then
20722072

20732073
if [ ! -d "${moduleDirPath}/${moduleAssetsDirName}" ]; then
20742074
mkdir -p "${moduleDirPath}/${moduleAssetsDirName}"
20752075
fi;
20762076

20772077
# pushd then cp to stop "cannot stat" error
2078-
pushd "${downloadDirPath}/${modulesDir}/${moduleDirName}" >/dev/null
2078+
pushd "${downloadModuleAssetsDirPath}/${moduleDirName}" >/dev/null
20792079

20802080
write "Moving contents of ${fileToGet} to ${moduleAssetsDirName}..."
2081-
# mv -f "${downloadDirPath}/${modulesDir}/${moduleDirName}/${moduleAssetsDirName}/*" "${moduleDirPath}/${moduleAssetsDirName}/"
2082-
# rsync --remove-source-files "${downloadDirPath}/${modulesDir}/${moduleDirName}/${moduleAssetsDirName}" "${moduleDirPath}/${moduleAssetsDirName}/"
2083-
move_recursive "${downloadDirPath}/${modulesDir}/${moduleDirName}/${moduleAssetsDirName}" "${moduleDirPath}/${moduleAssetsDirName}"
2084-
rm -rf "${downloadDirPath}/${modulesDir}/${moduleDirName}/${moduleAssetsDirName}"
2081+
# mv -f "${downloadModuleAssetsDirPath}/${moduleDirName}/${moduleAssetsDirName}/*" "${moduleDirPath}/${moduleAssetsDirName}/"
2082+
# rsync --remove-source-files "${downloadModuleAssetsDirPath}/${moduleDirName}/${moduleAssetsDirName}" "${moduleDirPath}/${moduleAssetsDirName}/"
2083+
move_recursive "${downloadModuleAssetsDirPath}/${moduleDirName}/${moduleAssetsDirName}" "${moduleDirPath}/${moduleAssetsDirName}"
2084+
rm -rf "${downloadModuleAssetsDirPath}/${moduleDirName}/${moduleAssetsDirName}"
20852085

20862086
if [ "$?" != "0" ]; then
20872087
writeLine "Failed." $color_error

‎src/server/appsettings.development.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
"DemoModulesDirPath": "%ROOT_PATH%/src/demos/modules",
3131
"ExternalModulesDirPath": "%ROOT_PATH%/../CodeProject.AI-Modules", // ** %EXTERNAL_MODULES_PATH% will expand to this value
3232

33-
"DownloadedModulePackagesDirPath": "%ROOT_PATH%/downloads/modules",
33+
"DownloadedModulePackagesDirPath": "%ROOT_PATH%/downloads/modules/packages",
3434
"DownloadedModelsPackagesDirPath": "%ROOT_PATH%/downloads/models",
3535

3636
// TO TEST MODULE / MODEL *DOWNLOAD* LOCALLY:
@@ -48,7 +48,7 @@
4848
// To test module install safely, specify install folders separate from your working folders
4949
// "RuntimesDirPath": "%ROOT_PATH%/debug/runtimes",
5050
// "ModulesDirPath": "%ROOT_PATH%/debug/modules",
51-
// "DownloadedModulePackagesDirPath": "%ROOT_PATH%/debug/downloads/modules",
51+
// "DownloadedModulePackagesDirPath": "%ROOT_PATH%/debug/downloads/modules/packages",
5252
// "ModuleInstallerScriptsDirPath": "%ROOT_PATH%/debug",
5353
// "PreInstalledModulesDirPath": "%ROOT_PATH%/debug/preinstalled-modules",
5454
// "DemoModulesDirPath": "%ROOT_PATH%/debug/demos/modules",

‎src/server/appsettings.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,8 +111,8 @@
111111
"ExternalModulesDirPath": "%ROOT_PATH%/../CodeProject.AI-Modules", // ** %EXTERNAL_MODULES_PATH% will expand to this value
112112

113113
// Downloads for modules and models go here
114-
"DownloadedModulePackagesDirPath": "%ROOT_PATH%/downloads/modules", // Where module packages are downloaded to
115-
"DownloadedModelsPackagesDirPath": "%ROOT_PATH%/downloads/models", // Where models are downloaded to
114+
"DownloadedModulePackagesDirPath": "%ROOT_PATH%/downloads/modules/packages", // Where module packages are downloaded to
115+
"DownloadedModelsPackagesDirPath": "%ROOT_PATH%/downloads/models", // Where models are downloaded to
116116

117117
// The template location of the Python interpreter for use when the Runtime for a module is
118118
// set as 'Python'. In Development mode, the Python folder will go under an OS specific

‎src/server/appsettings.linux.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
need to, plus it potentially means permission issues. For Docker we just
66
map paths, so no drama there.
77

8-
"DownloadedModulePackagesDirPath": "/opt/CodeProject/AI/downloads/modules",
8+
"DownloadedModulePackagesDirPath": "/opt/CodeProject/AI/downloads/modules/packages",
99
"ModulesDirPath": "/opt/CodeProject/AI/modules",
1010
*/
1111

‎src/setup.bat‎

Lines changed: 28 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -121,19 +121,15 @@ set appRootDirPath=!setupScriptDirPath!
121121
:: The location of large packages that need to be downloaded (eg an AWS S3 bucket
122122
:: name). This will be overwritten using the value from appsettings.json
123123
REM set assetStorageUrl=https://codeproject-ai.s3.ca-central-1.amazonaws.com/server/assets/
124-
REM set assetStorageUrl=https://codeproject-ai-bunny.b-cdn.net/server/assets/
125-
set assetStorageUrl=https://www.codeproject.com/ai/download/server/assets/
124+
set assetStorageUrl=https://codeproject-ai-bunny.b-cdn.net/server/assets/
125+
REM set assetStorageUrl=https://www.codeproject.com/ai/download/server/assets/
126126

127127
:: The name of the source directory (in development)
128128
set srcDirName=src
129129

130130
:: The name of the app directory (in docker)
131131
set appDirName=app
132132

133-
:: The name of the dir, within the current directory, where install assets will
134-
:: be downloaded
135-
set downloadDir=downloads
136-
137133
:: The name of the dir holding the installed runtimes
138134
set runtimesDir=runtimes
139135

@@ -142,6 +138,14 @@ set modulesDir=modules
142138
set preInstalledModulesDir=preinstalled-modules
143139
set externalModulesDir=CodeProject.AI-Modules
144140

141+
:: The name of the dir, relative to the root directory, containing the folder
142+
:: where downloaded assets will be cached
143+
set downloadDir=downloads
144+
145+
:: Name of the install assets folder. Downloads in <root>/downloads/modules/assets
146+
:: Module packages will be stored in <root>/downloads/modules/packages
147+
set assetsDir=assets
148+
145149
:: The name of the dir holding downloaded models for the modules. NOTE: this is
146150
:: not currently used, but here for future-proofing
147151
set modelsDir=models
@@ -257,6 +261,7 @@ set preInstalledModulesDirPath=!rootDirPath!\!preInstalledModulesDir!
257261
set externalModulesDirPath=!rootDirPath!\..\!externalModulesDir!
258262
set modelsDirPath=!rootDirPath!\!modelsDir!
259263
set downloadDirPath=!rootDirPath!\!downloadDir!
264+
set downloadModuleAssetsDirPath=!downloadDirPath!\!modulesDir!\!assetsDir!
260265
set utilsScriptsDirPath=!appRootDirPath!\scripts
261266
set installScriptsDirPath=!rootDirPath!\devops\install
262267
set utilsScript=!utilsScriptsDirPath!\utils.bat
@@ -367,7 +372,7 @@ if "!jsonFileValue!" NEQ "" set assetStorageUrl=!jsonFileValue!
367372
call "!utilsScript!" Write "Creating Directories..."
368373
if not exist "!runtimesDirPath!\" mkdir "!runtimesDirPath!"
369374
if not exist "!downloadDirPath!\" mkdir "!downloadDirPath!"
370-
if not exist "!downloadDirPath!\!modulesDir!\" mkdir "!downloadDirPath!\!modulesDir!\"
375+
if not exist "!downloadModuleAssetsDirPath!\" mkdir "!downloadModuleAssetsDirPath!\"
371376
if not exist "!downloadDirPath!\!modelsDir!\" mkdir "!downloadDirPath!\!modelsDir!\"
372377

373378
call "!utilsScript!" WriteLine "done" "Green"
@@ -376,21 +381,22 @@ call "!utilsScript!" WriteLine ""
376381
:: Output settings
377382
if /i "%verbosity%" neq "quiet" (
378383
call "!utilsScript!" WriteLine
379-
call "!utilsScript!" WriteLine "os, name, arch = !os! !os_name! !architecture!" !color_mute!
380-
call "!utilsScript!" WriteLine "systemName, platform = !systemName!, !platform!" !color_mute!
381-
call "!utilsScript!" WriteLine "edgeDevice = !edgeDevice!" !color_mute!
382-
call "!utilsScript!" WriteLine "setupMode = !setupMode!" !color_mute!
383-
call "!utilsScript!" WriteLine "executionEnvironment = !executionEnvironment!" !color_mute!
384-
call "!utilsScript!" WriteLine "rootDirPath = !rootDirPath!" !color_mute!
385-
call "!utilsScript!" WriteLine "appRootDirPath = !appRootDirPath!" !color_mute!
386-
call "!utilsScript!" WriteLine "setupScriptDirPath = !setupScriptDirPath!" !color_mute!
387-
call "!utilsScript!" WriteLine "utilsScriptsDirPath = !utilsScriptsDirPath!" !color_mute!
388-
call "!utilsScript!" WriteLine "runtimesDirPath = !runtimesDirPath!" !color_mute!
389-
call "!utilsScript!" WriteLine "modulesDirPath = !modulesDirPath!" !color_mute!
390-
call "!utilsScript!" WriteLine "externalModulesDirPath = !externalModulesDirPath!" !color_mute!
391-
call "!utilsScript!" WriteLine "modelsDirPath = !modelsDirPath!" !color_mute!
392-
call "!utilsScript!" WriteLine "downloadDirPath = !downloadDirPath!" !color_mute!
393-
call "!utilsScript!" WriteLine "assetStorageUrl = !assetStorageUrl!" !color_mute!
384+
call "!utilsScript!" WriteLine "os, name, arch = !os! !os_name! !architecture!" !color_mute!
385+
call "!utilsScript!" WriteLine "systemName, platform = !systemName!, !platform!" !color_mute!
386+
call "!utilsScript!" WriteLine "edgeDevice = !edgeDevice!" !color_mute!
387+
call "!utilsScript!" WriteLine "setupMode = !setupMode!" !color_mute!
388+
call "!utilsScript!" WriteLine "executionEnvironment = !executionEnvironment!" !color_mute!
389+
call "!utilsScript!" WriteLine "rootDirPath = !rootDirPath!" !color_mute!
390+
call "!utilsScript!" WriteLine "appRootDirPath = !appRootDirPath!" !color_mute!
391+
call "!utilsScript!" WriteLine "setupScriptDirPath = !setupScriptDirPath!" !color_mute!
392+
call "!utilsScript!" WriteLine "utilsScriptsDirPath = !utilsScriptsDirPath!" !color_mute!
393+
call "!utilsScript!" WriteLine "runtimesDirPath = !runtimesDirPath!" !color_mute!
394+
call "!utilsScript!" WriteLine "modulesDirPath = !modulesDirPath!" !color_mute!
395+
call "!utilsScript!" WriteLine "externalModulesDirPath = !externalModulesDirPath!" !color_mute!
396+
call "!utilsScript!" WriteLine "modelsDirPath = !modelsDirPath!" !color_mute!
397+
call "!utilsScript!" WriteLine "downloadDirPath = !downloadDirPath!" !color_mute!
398+
call "!utilsScript!" WriteLine "downloadModuleAssetsDirPath = !downloadModuleAssetsDirPath!" !color_mute!
399+
call "!utilsScript!" WriteLine "assetStorageUrl = !assetStorageUrl!" !color_mute!
394400
call "!utilsScript!" WriteLine
395401
)
396402

‎src/setup.sh‎

Lines changed: 33 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -138,19 +138,15 @@ appRootDirPath="${setupScriptDirPath}"
138138
# The location of large packages that need to be downloaded (eg an AWS S3 bucket
139139
# name). This will be overwritten using the value from appsettings.json
140140
# assetStorageUrl='https://codeproject-ai.s3.ca-central-1.amazonaws.com/server/assets/'
141-
# assetStorageUrl='https://codeproject-ai-bunny.b-cdn.net/server/assets/'
142-
assetStorageUrl='https://www.codeproject.com/ai/download/server/assets/'
141+
assetStorageUrl='https://codeproject-ai-bunny.b-cdn.net/server/assets/'
142+
# assetStorageUrl='https://www.codeproject.com/ai/download/server/assets/'
143143

144144
# The name of the source directory (in development)
145145
srcDirName='src'
146146

147147
# The name of the app directory (in docker)
148148
appDirName='app'
149149

150-
# The name of the dir, within the current directory, where install assets will
151-
# be downloaded
152-
downloadDir='downloads'
153-
154150
# The name of the dir holding the runtimes
155151
runtimesDir='runtimes'
156152

@@ -161,6 +157,14 @@ modulesDir="modules"
161157
preInstalledModulesDir="preinstalled-modules"
162158
externalModulesDir="CodeProject.AI-Modules"
163159

160+
# The name of the dir, relative to the root directory, containing the folder
161+
# where downloaded assets will be cached
162+
downloadDir='downloads'
163+
164+
# Name of the install assets folder. Downloads in <root>/downloads/modules/assets
165+
# Module packages will be stored in <root>/downloads/modules/packages
166+
assetsDir='assets'
167+
164168
# The name of the dir holding downloaded models for the modules. NOTE: this is
165169
# not currently used, but here for future-proofing
166170
modelsDir="models"
@@ -304,6 +308,7 @@ preInstalledModulesDirPath="${rootDirPath}/${preInstalledModulesDir}"
304308
externalModulesDirPath="${rootDirPath}/../${externalModulesDir}"
305309
modelsDirPath="${rootDirPath}/${modelsDir}"
306310
downloadDirPath="${rootDirPath}/${downloadDir}"
311+
downloadModuleAssetsDirPath="${downloadDirPath}/${modulesDir}/${assetsDir}"
307312
utilsScriptsDirPath="${appRootDirPath}/scripts"
308313
installScriptsDirPath="${rootDirPath}/devops/install"
309314
utilsScript="${utilsScriptsDirPath}/utils.sh"
@@ -814,33 +819,35 @@ if [ "$serverStorageUrl" != "" ]; then assetStorageUrl="$serverStorageUrl"; fi
814819

815820
# Create some directories (run under a subshell, all with sudo)
816821

817-
CreateWriteableDir "${runtimesDirPath}" "runtimes"
818-
CreateWriteableDir "${downloadDirPath}" "downloads"
819-
CreateWriteableDir "${modulesDirPath}" "modules download"
820-
CreateWriteableDir "${modelsDirPath}" "models download"
821-
CreateWriteableDir "${commonDataDirPath}" "persisted data"
822+
CreateWriteableDir "${runtimesDirPath}" "runtimes"
823+
CreateWriteableDir "${downloadDirPath}" "general downloads"
824+
CreateWriteableDir "${downloadModuleAssetsDirPath}" "module asset downloads"
825+
CreateWriteableDir "${modulesDirPath}" "modules"
826+
CreateWriteableDir "${modelsDirPath}" "models"
827+
CreateWriteableDir "${commonDataDirPath}" "persisted data"
822828

823829
writeLine
824830

825831
# Output settings
826832

827833
if [ "$verbosity" != "quiet" ]; then
828834
writeLine
829-
writeLine "os, name, arch = ${os} ${os_name} ${architecture}" $color_mute
830-
writeLine "systemName, platform = ${systemName}, ${platform}" $color_mute
831-
writeLine "edgeDevice = ${edgeDevice}" $color_mute
832-
writeLine "SSH = ${inSSH}" $color_mute
833-
writeLine "setupMode = ${setupMode}" $color_mute
834-
writeLine "executionEnvironment = ${executionEnvironment}" $color_mute
835-
writeLine "rootDirPath = ${rootDirPath}" $color_mute
836-
writeLine "appRootDirPath = ${appRootDirPath}" $color_mute
837-
writeLine "setupScriptDirPath = ${setupScriptDirPath}" $color_mute
838-
writeLine "utilsScriptsDirPath = ${utilsScriptsDirPath}" $color_mute
839-
writeLine "runtimesDirPath = ${runtimesDirPath}" $color_mute
840-
writeLine "modulesDirPath = ${modulesDirPath}" $color_mute
841-
writeLine "externalModulesDirPath = ${externalModulesDirPath}" $color_mute
842-
writeLine "downloadDirPath = ${downloadDirPath}" $color_mute
843-
writeLine "assetStorageUrl = ${assetStorageUrl}" $color_mute
835+
writeLine "os, name, arch = ${os} ${os_name} ${architecture}" $color_mute
836+
writeLine "systemName, platform = ${systemName}, ${platform}" $color_mute
837+
writeLine "edgeDevice = ${edgeDevice}" $color_mute
838+
writeLine "SSH = ${inSSH}" $color_mute
839+
writeLine "setupMode = ${setupMode}" $color_mute
840+
writeLine "executionEnvironment = ${executionEnvironment}" $color_mute
841+
writeLine "rootDirPath = ${rootDirPath}" $color_mute
842+
writeLine "appRootDirPath = ${appRootDirPath}" $color_mute
843+
writeLine "setupScriptDirPath = ${setupScriptDirPath}" $color_mute
844+
writeLine "utilsScriptsDirPath = ${utilsScriptsDirPath}" $color_mute
845+
writeLine "runtimesDirPath = ${runtimesDirPath}" $color_mute
846+
writeLine "modulesDirPath = ${modulesDirPath}" $color_mute
847+
writeLine "externalModulesDirPath = ${externalModulesDirPath}" $color_mute
848+
writeLine "downloadDirPath = ${downloadDirPath}" $color_mute
849+
writeLine "downloadModuleAssetsDirPath = ${downloadModuleAssetsDirPath}" $color_mute
850+
writeLine "assetStorageUrl = ${assetStorageUrl}" $color_mute
844851
writeLine
845852
fi
846853

0 commit comments

Comments
 (0)