Skip to content

Commit 0a0cbc2

Browse files
alan-nullalan-null
committed
Added pipelines for intermediate operations
- message json processing - file json processing - file raw-data processing
1 parent 930a43c commit 0a0cbc2

10 files changed

+358
-2
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Tests/Output/

‎Slack.Backup/Private/backup/Invoke-ChannelBackup.ps1‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ function Invoke-ChannelBackup {
4848
}
4949

5050
if ($newMessages.Count -gt 0) {
51-
$messages += $newMessages
51+
$messages += $newMessages | % { Invoke-MessageProcessingPipeline -SlackMessage $_ }
5252
$messages | Sort-Object -Property @{Expression = { [double]$_.ts } } | ConvertTo-Json -Depth 100 | Set-Content -Path $Location
5353
}
5454
}

‎Slack.Backup/Private/backup/Invoke-FilesBackup.ps1‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,9 @@ function Invoke-FilesBackup {
2929
$ext = $f.filetype
3030
$path = "$backupLoc/$($f.id)"
3131
$slackFile = Get-SlackFile -Token $Token -Uri $f.url_private
32+
$slackFile = Invoke-FileDataProcessingPipeline -SlackFile $slackFile
3233
[System.IO.File]::WriteAllBytes("$path.$ext", $slackFile)
33-
$f | ConvertTo-Json -Depth 10 | Set-Content -Path "$path.json"
34+
$f | Invoke-FileProcessingPipeline | ConvertTo-Json -Depth 10 | Set-Content -Path "$path.json"
3435
}
3536
}
3637
}

‎Slack.Backup/Private/pipelines/Invoke-FileDataProcessingPipeline.ps1‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
function Invoke-FileDataProcessingPipeline {
2+
param (
3+
[Parameter(Mandatory = $true, Position = 0, ValueFromPipeline = $true )]
4+
[byte[]]$SlackFile
5+
)
6+
process {
7+
Get-Command -Name Invoke-FileDataProcessor* -CommandType Alias | `
8+
Sort-Object -Property Name | `
9+
? { $_.Parameters.Count -gt 0 } | `
10+
? { $_.Parameters.ContainsKey("SlackFile") } | `
11+
? { $_.Parameters['SlackFile'].ParameterType.Name -eq [byte[]] } | `
12+
% {
13+
Write-Verbose "Invoking processor: $($_.Name)"
14+
$SlackFile = &($_.Name) -SlackFile $SlackFile
15+
}
16+
$SlackFile
17+
}
18+
}

‎Slack.Backup/Private/pipelines/Invoke-FileProcessingPipeline.ps1‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
function Invoke-FileProcessingPipeline {
2+
param (
3+
[Parameter(Mandatory = $true, Position = 0, ValueFromPipeline = $true )]
4+
[System.Object]$SlackFile
5+
)
6+
process {
7+
Get-Command -Name Invoke-FileProcessor* -CommandType Alias | `
8+
Sort-Object -Property Name | `
9+
? { $_.Parameters.Count -gt 0 } | `
10+
? { $_.Parameters.ContainsKey("SlackFile") } | `
11+
? { $_.Parameters['SlackFile'].ParameterType.Name -eq "Object" } | `
12+
% {
13+
Write-Verbose "Invoking processor: $($_.Name)"
14+
$SlackFile = &($_.Name) -SlackFile $SlackFile
15+
}
16+
$SlackFile
17+
}
18+
}

‎Slack.Backup/Private/pipelines/Invoke-MessageProcessingPipeline.ps1‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
function Invoke-MessageProcessingPipeline {
2+
param (
3+
[Parameter(Mandatory = $true, Position = 0, ValueFromPipeline = $true )]
4+
[System.Object]$SlackMessage
5+
)
6+
process {
7+
Get-Command -Name Invoke-MessageProcessor* -CommandType Alias | `
8+
Sort-Object -Property Name | `
9+
? { $_.Parameters.ContainsKey("SlackMessage") } | `
10+
? { $_.Parameters['SlackMessage'].ParameterType.Name -eq "Object" } | `
11+
% {
12+
Write-Verbose "Invoking processor: $($_.Name)"
13+
$SlackMessage = &($_.Name) -SlackMessage $SlackMessage
14+
}
15+
$SlackMessage
16+
}
17+
}

‎Tests/MockData/conversations.history.json‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
{
2+
"ok": true,
3+
"messages": [
4+
{
5+
"type": "message",
6+
"user": "U012AB3CDE",
7+
"text": "I find you punny and would like to smell your nose letter",
8+
"ts": "1512085950.000216"
9+
},
10+
{
11+
"type": "message",
12+
"user": "U061F7AUR",
13+
"text": "What, you want to smell my shoes better?",
14+
"ts": "1512104434.000490"
15+
}
16+
],
17+
"has_more": true,
18+
"pin_count": 0,
19+
"response_metadata": {
20+
"next_cursor": "bmV4dF90czoxNTEyMDg1ODYxMDAwNTQz"
21+
}
22+
}

‎Tests/MockData/conversations.list.json‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
{
2+
"ok": true,
3+
"channels": [
4+
{
5+
"id": "C012AB3CD",
6+
"name": "general",
7+
"is_channel": true,
8+
"is_group": false,
9+
"is_im": false,
10+
"created": 1449252889,
11+
"creator": "U012A3CDE",
12+
"is_archived": false,
13+
"is_general": true,
14+
"unlinked": 0,
15+
"name_normalized": "general",
16+
"is_shared": false,
17+
"is_ext_shared": false,
18+
"is_org_shared": false,
19+
"pending_shared": [],
20+
"is_pending_ext_shared": false,
21+
"is_member": true,
22+
"is_private": false,
23+
"is_mpim": false,
24+
"topic": {
25+
"value": "Company-wide announcements and work-based matters",
26+
"creator": "",
27+
"last_set": 0
28+
},
29+
"purpose": {
30+
"value": "This channel is for team-wide communication and announcements. All team members are in this channel.",
31+
"creator": "",
32+
"last_set": 0
33+
},
34+
"previous_names": [],
35+
"num_members": 4
36+
},
37+
{
38+
"id": "C061EG9T2",
39+
"name": "random",
40+
"is_channel": true,
41+
"is_group": false,
42+
"is_im": false,
43+
"created": 1449252889,
44+
"creator": "U061F7AUR",
45+
"is_archived": false,
46+
"is_general": false,
47+
"unlinked": 0,
48+
"name_normalized": "random",
49+
"is_shared": false,
50+
"is_ext_shared": false,
51+
"is_org_shared": false,
52+
"pending_shared": [],
53+
"is_pending_ext_shared": false,
54+
"is_member": true,
55+
"is_private": false,
56+
"is_mpim": false,
57+
"topic": {
58+
"value": "Non-work banter and water cooler conversation",
59+
"creator": "",
60+
"last_set": 0
61+
},
62+
"purpose": {
63+
"value": "A place for non-work-related flimflam, faffing, hodge-podge or jibber-jabber you'd prefer to keep out of more focused work-related channels.",
64+
"creator": "",
65+
"last_set": 0
66+
},
67+
"previous_names": [],
68+
"num_members": 4
69+
}
70+
],
71+
"response_metadata": {
72+
"next_cursor": "dGVhbTpDMDYxRkE1UEI="
73+
}
74+
}

‎Tests/MockData/files.list.json‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
{
2+
"ok": true,
3+
"files": [
4+
{
5+
"id": "F0S43P1CZ",
6+
"created": 1531763254,
7+
"timestamp": 1531763254,
8+
"name": "billair.gif",
9+
"title": "billair.gif",
10+
"mimetype": "image/gif",
11+
"filetype": "gif",
12+
"pretty_type": "GIF",
13+
"user": "U061F7AUR",
14+
"editable": false,
15+
"size": 144538,
16+
"mode": "hosted",
17+
"is_external": false,
18+
"external_type": "",
19+
"is_public": true,
20+
"public_url_shared": false,
21+
"display_as_bot": false,
22+
"username": "",
23+
"url_private": "https://.../billair.gif",
24+
"url_private_download": "https://.../billair.gif",
25+
"thumb_64": "https://.../billair_64.png",
26+
"thumb_80": "https://.../billair_80.png",
27+
"thumb_360": "https://.../billair_360.png",
28+
"thumb_360_w": 176,
29+
"thumb_360_h": 226,
30+
"thumb_160": "https://.../billair_=_160.png",
31+
"thumb_360_gif": "https://.../billair_360.gif",
32+
"image_exif_rotation": 1,
33+
"original_w": 176,
34+
"original_h": 226,
35+
"deanimate_gif": "https://.../billair_deanimate_gif.png",
36+
"pjpeg": "https://.../billair_pjpeg.jpg",
37+
"permalink": "https://.../billair.gif",
38+
"permalink_public": "https://.../...",
39+
"channels": [
40+
"C0T8SE4AU"
41+
],
42+
"groups": [],
43+
"ims": [],
44+
"comments_count": 0
45+
},
46+
{
47+
"id": "F0S43PZDF",
48+
"created": 1531763342,
49+
"timestamp": 1531763342,
50+
"name": "tedair.gif",
51+
"title": "tedair.gif",
52+
"mimetype": "image/gif",
53+
"filetype": "gif",
54+
"pretty_type": "GIF",
55+
"user": "U061F7AUR",
56+
"editable": false,
57+
"size": 137531,
58+
"mode": "hosted",
59+
"is_external": false,
60+
"external_type": "",
61+
"is_public": true,
62+
"public_url_shared": false,
63+
"display_as_bot": false,
64+
"username": "",
65+
"url_private": "https://.../tedair.gif",
66+
"url_private_download": "https://.../tedair.gif",
67+
"thumb_64": "https://.../tedair_64.png",
68+
"thumb_80": "https://.../tedair_80.png",
69+
"thumb_360": "https://.../tedair_360.png",
70+
"thumb_360_w": 176,
71+
"thumb_360_h": 226,
72+
"thumb_160": "https://.../tedair_=_160.png",
73+
"thumb_360_gif": "https://.../tedair_360.gif",
74+
"image_exif_rotation": 1,
75+
"original_w": 176,
76+
"original_h": 226,
77+
"deanimate_gif": "https://.../tedair_deanimate_gif.png",
78+
"pjpeg": "https://.../tedair_pjpeg.jpg",
79+
"permalink": "https://.../tedair.gif",
80+
"permalink_public": "https://.../...",
81+
"channels": [
82+
"C0T8SE4AU"
83+
],
84+
"groups": [],
85+
"ims": [],
86+
"comments_count": 0
87+
}
88+
],
89+
"paging": {
90+
"count": 100,
91+
"total": 2,
92+
"page": 1,
93+
"pages": 1
94+
}
95+
}

0 commit comments

Comments
 (0)