Skip to content

Commit c8ebf4d

Browse files
committed
Feature/versioned packages (#3496)
* WIP, moved to AssemblyRewriter for the merging of the main packages and introduced versioned packages on the canary build * Cleaned up targets and made versioned nuget packages only a thing when doing canary builds * allow a project to signal it wants to rewrite the dependencies but not merge them into the final dll, needed for Nest.JsonNetSerializer * fail on first error when testing nuget packages, do not rewrite json.net for Nest.JsonNetSerializer, remove publish from canary target * remove -stoponfail, should work but does not * make sure published projects do not end up being merged but stay referenced through nuget * update to latest assembly-resolver * fix code standards tests to reflect Elastic.Internal namespace * Versioned Nuget packages are now more distinct * make sure yml uses spaces, only publish artifacts on master and \d.x branches, update to latest assemblyrewriter (we had this repos as 2 different projects on appveyor (cherry picked from commit 67a2482)
1 parent 662f963 commit c8ebf4d

16 files changed

+281
-266
lines changed

.editorconfig

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ indent_size = 4
1212
indent_style = tab
1313
indent_size = 4
1414

15-
[*.{fs,fsx}]
15+
[*.{fs,fsx,yml}]
1616
indent_style = space
1717
indent_size = 4
1818

appveyor.yml

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,20 @@
11
version: 1.0.{build}
22
image: Visual Studio 2017
3-
build_script:
4-
- cmd: build.bat canary skipdocs
53
test: off
64
environment:
7-
DOTNET_CLI_TELEMETRY_OPTOUT: true
8-
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
5+
DOTNET_CLI_TELEMETRY_OPTOUT: true
6+
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
7+
build_script:
8+
- cmd: build.bat canary skipdocs
9+
10+
nuget:
11+
disable_publish_on_pr: true
12+
13+
for:
14+
15+
- branches:
16+
only:
17+
- /^(master|\d+\.x)$/
18+
artifacts:
19+
- path: .\build\output\_packages\*.nupkg
20+
name: NuGet

build/NEST.nuspec

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,21 @@
44
<id>NEST</id>
55
<version>$version$</version>
66
<title>NEST - Elasticsearch .NET high level client</title>
7+
78
<authors>Elastic and contributors</authors>
89
<owners>Elastic</owners>
10+
<copyright>2014-$year$ Elasticsearch BV</copyright>
11+
12+
<requireLicenseAcceptance>false</requireLicenseAcceptance>
913
<licenseUrl>https://github.com/elastic/elasticsearch-net/blob/master/license.txt</licenseUrl>
1014
<projectUrl>https://github.com/elastic/elasticsearch-net</projectUrl>
1115
<iconUrl>https://raw.githubusercontent.com/elastic/elasticsearch-net/master/build/nuget-icon.png</iconUrl>
12-
<requireLicenseAcceptance>false</requireLicenseAcceptance>
16+
1317
<description>Strongly typed interface to Elasticsearch. Fluent and classic object initializer mappings of requests and responses. Uses and exposes Elasticsearch.Net</description>
1418
<summary>Strongly typed interface to Elasticsearch. Fluent and classic object initializer mappings of requests and responses. Uses and exposes Elasticsearch.Net</summary>
15-
<releaseNotes>See https://github.com/elastic/elasticsearch-net/releases/tag/$version$</releaseNotes>
16-
<copyright>2014-$year$ Elasticsearch BV</copyright>
19+
<releaseNotes>https://github.com/elastic/elasticsearch-net/releases/tag/$version$</releaseNotes>
1720
<tags>elasticsearch,elastic,search,lucene,nest</tags>
21+
1822
<dependencies>
1923
<group targetFramework="net461">
2024
<dependency id="Elasticsearch.Net" version="[$version$, $nextMajorVersion$)" />

build/nuget-icon-aux.png

18.2 KB
Loading

build/scripts/Building.fsx

Lines changed: 36 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,8 @@
1212

1313
open System
1414
open System.IO
15-
open System.Reflection
1615
open Fake
1716
open FSharp.Data
18-
open Mono.Cecil
1917

2018
open Paths
2119
open Projects
@@ -78,100 +76,47 @@ module Build =
7876
CleanDir Paths.BuildOutput
7977
DotNetCli.RunCommand (fun p -> { p with TimeOut = TimeSpan.FromMinutes(5.) }) "clean src/Elasticsearch.sln -c Release" |> ignore
8078
DotNetProject.All |> Seq.iter(fun p -> CleanDir(Paths.BinFolder p.Name))
81-
82-
type CustomResolver(folder) =
83-
inherit DefaultAssemblyResolver()
84-
member this.Folder = folder;
85-
86-
override this.Resolve name =
87-
try
88-
base.Resolve name
89-
with
90-
| ex ->
91-
AssemblyDefinition.ReadAssembly(Path.Combine(folder, "Elasticsearch.Net.dll"));
92-
93-
let private rewriteNamespace nest f =
94-
trace "Rewriting namespaces"
95-
let folder = Paths.ProjectOutputFolder nest f
96-
let nestDll = sprintf "%s/%s.dll" folder nest.Name
97-
let nestRewrittenDll = sprintf "%s/%s-rewriten.dll" folder nest.Name
98-
use resolver = new CustomResolver(folder)
99-
let readerParams = ReaderParameters( AssemblyResolver = resolver, ReadWrite = true );
100-
use nestAssembly = AssemblyDefinition.ReadAssembly(nestDll, readerParams);
10179

102-
for item in nestAssembly.MainModule.Types do
103-
if item.Namespace.StartsWith("Newtonsoft.Json") then
104-
item.Namespace <- item.Namespace.Replace("Newtonsoft.Json", "Nest.Json")
105-
106-
// Touch custom attribute arguments
107-
// Cecil does not update the types referenced within these attributes automatically,
108-
// so enumerate them to ensure namespace renaming is reflected in these references.
109-
let touchAttributes (attributes:Mono.Collections.Generic.Collection<CustomAttribute>) =
110-
for attr in attributes do
111-
if attr.HasConstructorArguments then
112-
for constArg in attr.ConstructorArguments do
113-
if constArg.Type.Name = "Type" then ignore()
114-
115-
// rewrite explicitly implemented interface definitions defined
116-
// in Newtonsoft.Json
117-
let rewriteName (method:IMemberDefinition) =
118-
if method.Name.Contains("Newtonsoft.Json") then
119-
method.Name <- method.Name.Replace("Newtonsoft.Json", "Nest.Json")
120-
121-
// recurse through all types and nested types
122-
let rec rewriteTypes (types:Mono.Collections.Generic.Collection<TypeDefinition>) =
123-
for t in types do
124-
touchAttributes t.CustomAttributes
125-
for prop in t.Properties do
126-
touchAttributes prop.CustomAttributes
127-
rewriteName prop
128-
if prop.GetMethod <> null then rewriteName prop.GetMethod
129-
if prop.SetMethod <> null then rewriteName prop.SetMethod
130-
for method in t.Methods do
131-
touchAttributes method.CustomAttributes
132-
rewriteName method
133-
for over in method.Overrides do rewriteName method
134-
for field in t.Fields do touchAttributes field.CustomAttributes
135-
for interf in t.Interfaces do touchAttributes interf.CustomAttributes
136-
for event in t.Events do touchAttributes event.CustomAttributes
137-
if t.HasNestedTypes then rewriteTypes t.NestedTypes
138-
139-
nestAssembly.MainModule.Types |> rewriteTypes
80+
let private assemblyRewriter = Paths.PaketDotNetGlobalTool "assembly-rewriter" @"tools\netcoreapp2.1\any\assembly-rewriter.dll"
81+
let private keyFile = Paths.Keys "keypair.snk"
82+
let Rewrite version framework projects =
83+
let project = projects |> Seq.head
84+
let folder = Paths.ProjectOutputFolder project framework
14085

141-
let resources = nestAssembly.MainModule.Resources
142-
for i = resources.Count-1 downto 0 do
143-
let resource = resources.[i]
144-
// remove the Newtonsoft signing key
145-
if resource.Name = "Newtonsoft.Json.Dynamic.snk" then resources.Remove(resource) |> ignore
146-
147-
let key = File.ReadAllBytes(Paths.Keys("keypair.snk"))
148-
let kp = StrongNameKeyPair(key)
149-
let wp = WriterParameters ( StrongNameKeyPair = kp);
150-
nestAssembly.Write(wp) |> ignore;
151-
trace "Finished rewriting namespaces"
86+
let dllFullPath name = sprintf "%s/%s.dll" folder name
87+
let outputName (p: DotNetProject) = match p.Name = project.Name with | true -> p.Name | _ -> p.InternalName
88+
let fullOutput (p: DotNetProject) = dllFullPath (p.Versioned (outputName p) version)
89+
let dlls =
90+
projects
91+
|> Seq.map (fun p -> sprintf @"-i ""%s"" -o ""%s"" " (dllFullPath p.Name) (fullOutput p))
92+
|> Seq.fold (+) " "
93+
94+
let mergeCommand = sprintf @"%s %s" assemblyRewriter dlls
95+
DotNetCli.RunCommand (fun p -> { p with TimeOut = TimeSpan.FromMinutes(3.) }) mergeCommand |> ignore
96+
97+
let mergedOutFile = fullOutput project
98+
let ilMergeArgs = [
99+
"/internalize";
100+
(sprintf "/lib:%s" folder);
101+
(sprintf "/keyfile:%s" keyFile);
102+
(sprintf "/out:%s" mergedOutFile)
103+
]
104+
let mergeDlls =
105+
projects
106+
|> Seq.filter (fun p -> p.Name = project.Name || not <| (DotNetProject.AllPublishable |> Seq.contains p))
107+
|> Seq.map fullOutput
108+
match project.NeedsMerge with
109+
| true -> Tooling.ILRepack.Exec (ilMergeArgs |> Seq.append mergeDlls) |> ignore
110+
| _ -> Tooling.ILRepack.Exec (ilMergeArgs |> Seq.append [mergeDlls |> Seq.head]) |> ignore
152111

153112
let private ilRepackInternal() =
154113
let fw = DotNetFramework.All
155-
for f in fw do
156-
let nest = Project Project.Nest
157-
let folder = Paths.ProjectOutputFolder nest f
158-
let nestDll = sprintf "%s/%s.dll" folder nest.Name
159-
let nestMergedDll = sprintf "%s/%s-merged.dll" folder nest.Name
160-
let jsonDll = sprintf "%s/Newtonsoft.Json.dll" folder
161-
let keyFile = Paths.Keys("keypair.snk");
162-
let options =
163-
[
164-
"/keyfile:", keyFile;
165-
"/internalize", "";
166-
"/lib:", folder;
167-
"/out:", nestDll;
168-
]
169-
|> List.map (fun (p,v) -> sprintf "%s%s" p v)
170-
171-
let args = [nestDll; jsonDll;] |> List.append options;
172-
173-
Tooling.ILRepack.Exec args |> ignore
174-
rewriteNamespace nest f |> ignore
114+
let projects = DotNetProject.AllPublishable
115+
let currentMajor = sprintf "%i" <| Versioning.CurrentVersion.Major
116+
for f in fw do
117+
for p in projects do
118+
if p.VersionedMergeDependencies <> [] then Rewrite (Some currentMajor) f p.VersionedMergeDependencies
119+
if p.MergeDependencies <> [] then Rewrite None f p.MergeDependencies
175120

176121
let ILRepack() =
177122
//nothing to IL merge in the 5.x branch

build/scripts/Commandline.fsx

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,8 @@ Targets:
2525
* integrate <elasticsearch_versions> [clustername] [testfilter]
2626
- run integration tests for <elasticsearch_versions> which is a semicolon separated list of
2727
elasticsearch versions to test or `latest`. Can filter tests by <clustername> and <testfilter>
28-
* canary [apikey] [feed]
29-
- create a canary nuget package based on the current version if [feed] and [apikey] are provided
30-
also pushes to upstream (myget)
28+
* canary
29+
- create a canary nuget package based on the current version.
3130
* diff <github|nuget|dir|assembly> <version|path 1> <version|path 2> [format]
3231
* cluster <cluster-name> [version]
3332
- Start a cluster defined in Tests.Core or Tests from the command line and leaves it running
@@ -220,12 +219,6 @@ module Commandline =
220219
| ["touch"; ] -> ignore()
221220
| ["temp"; ] -> ignore()
222221
| ["canary"; ] -> ignore()
223-
| ["canary"; apiKey ] ->
224-
setBuildParam "apiKey" apiKey
225-
setBuildParam "feed" "elasticsearch-net"
226-
| ["canary"; apiKey; feed ] ->
227-
setBuildParam "apiKey" apiKey
228-
setBuildParam "feed" feed
229222
| _ ->
230223
traceError usage
231224
exit 2

build/scripts/Differ.fsx

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,7 @@ open System
1111
open System.IO
1212

1313
open Fake
14-
open System
15-
open System.IO
1614
open System.Linq
17-
open System.Net
18-
open System.Text
1915
open System.Text.RegularExpressions
2016
open System.Xml
2117
open System.Xml.Linq

build/scripts/Paths.fsx

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ module Paths =
2727
let SourceFolder = "src"
2828

2929
let CheckedInTool(tool) = sprintf "%s/%s" CheckedInToolsFolder tool
30+
let PaketDotNetGlobalTool tool subPath = sprintf "%s/%s" (Tool tool) subPath
3031
let Keys(keyFile) = sprintf "%s/%s" KeysFolder keyFile
3132
let Output(folder) = sprintf "%s/%s" BuildOutput folder
3233
let Source(folder) = sprintf "%s/%s" SourceFolder folder
@@ -40,7 +41,7 @@ module Paths =
4041
match p with
4142
| Tests -> sprintf "%s/%s/%s.csproj" SourceFolder project.Name project.Name
4243
| DocGenerator -> sprintf "%s/CodeGeneration/%s/%s.csproj" SourceFolder project.Name project.Name
43-
44+
| _ -> null
4445

4546
let BinFolder(folder) =
4647
let f = replace @"\" "/" folder

build/scripts/Projects.fsx

Lines changed: 35 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,14 @@ module Projects =
3131
type PrivateProject =
3232
| Tests
3333
| DocGenerator
34+
35+
type DependencyProject =
36+
| JsonNet
3437

3538
type DotNetProject =
3639
| Project of Project
3740
| PrivateProject of PrivateProject
41+
| DepencyProject of DependencyProject
3842

3943
static member All =
4044
seq [
@@ -46,25 +50,41 @@ module Projects =
4650
static member AllPublishable = seq [Project Project.ElasticsearchNet; Project Project.Nest;]
4751
static member Tests = seq [PrivateProject PrivateProject.Tests;]
4852

53+
member this.MergeDependencies=
54+
match this with
55+
| Project Nest -> [Project Project.Nest; DepencyProject DependencyProject.JsonNet]
56+
| _ -> []
57+
58+
member this.VersionedMergeDependencies =
59+
match this with
60+
| Project Nest -> [Project Project.Nest; Project Project.ElasticsearchNet; DepencyProject DependencyProject.JsonNet]
61+
| Project NestJsonNetSerializer -> [Project NestJsonNetSerializer; Project Project.Nest; Project Project.ElasticsearchNet ]
62+
| Project ElasticsearchNet -> [Project ElasticsearchNet]
63+
| _ -> []
64+
4965
member this.Name =
5066
match this with
51-
| Project p ->
52-
match p with
53-
| Nest -> "Nest"
54-
| ElasticsearchNet -> "Elasticsearch.Net"
55-
| PrivateProject p ->
56-
match p with
57-
| Tests -> "Tests"
58-
| DocGenerator -> "DocGenerator"
67+
| Project Nest -> "Nest"
68+
| Project ElasticsearchNet -> "Elasticsearch.Net"
69+
| PrivateProject Tests -> "Tests"
70+
| PrivateProject DocGenerator -> "DocGenerator"
71+
| DepencyProject JsonNet -> "Newtonsoft.Json"
72+
73+
member this.NugetId = match this with | Project Nest -> "NEST" | _ -> this.Name
74+
75+
member this.NeedsMerge = match this with | Project NestJsonNetSerializer -> false | _ -> true
5976

60-
member this.Nuspec =
77+
member this.Versioned name version =
78+
match version with
79+
| Some s -> sprintf "%s%s" name s
80+
| None -> name
81+
82+
member this.InternalName =
6183
match this with
62-
| Project p ->
63-
match p with
64-
| Nest -> "NEST"
65-
| _ -> this.Name
66-
| _ -> this.Name
67-
84+
| Project p -> this.Name
85+
| PrivateProject p -> sprintf "Elastic.Internal.%s" this.Name
86+
| DepencyProject JsonNet -> "Elastic.Internal.JsonNet"
87+
6888
static member TryFindName (name: string) =
6989
DotNetProject.All
7090
|> Seq.map(fun p -> p.Name)

0 commit comments

Comments
 (0)