Skip to content

Commit 48d9700

Browse files
1vn1vn
authored
Make schema validation optional (#100)
1 parent 9ceed0c commit 48d9700

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+459
-201
lines changed

‎examples/mnist/resources/transformed_columns.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
- kind: transformed_column
22
name: image_pixels
3-
transformer: decode_and_normalize
3+
transformer_path: implementations/transformers/decode_and_normalize.py
44
inputs:
55
columns:
66
image: image

‎examples/mnist/resources/transformers.yaml‎

Lines changed: 0 additions & 6 deletions
This file was deleted.

‎examples/reviews/resources/max_length.yaml‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,6 @@
1-
- kind: aggregator
2-
name: max_length
3-
inputs:
4-
columns:
5-
col: STRING_COLUMN
6-
output_type: INT
7-
81
- kind: aggregate
92
name: max_review_length
10-
aggregator: max_length
3+
aggregator_path: implementations/aggregators/max_length.py
114
inputs:
125
columns:
136
col: review

‎examples/reviews/resources/tokenized_columns.yaml‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,6 @@
1-
- kind: transformer
2-
name: tokenize_string_to_int
3-
output_type: INT_LIST_COLUMN
4-
inputs:
5-
columns:
6-
col: STRING_COLUMN
7-
args:
8-
max_len: INT
9-
vocab: {STRING: INT}
10-
111
- kind: transformed_column
122
name: embedding_input
13-
transformer: tokenize_string_to_int
3+
transformer_path: implementations/transformers/tokenize_string_to_int.py
144
inputs:
155
columns:
166
col: review

‎pkg/consts/consts.go‎

Lines changed: 18 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -36,21 +36,24 @@ var (
3636
RequirementsTxt = "requirements.txt"
3737
PackageDir = "packages"
3838

39-
AppsDir = "apps"
40-
DataDir = "data"
41-
RawDataDir = "data_raw"
42-
TrainingDataDir = "data_training"
43-
AggregatorsDir = "aggregators"
44-
AggregatesDir = "aggregates"
45-
TransformersDir = "transformers"
46-
ModelImplsDir = "model_implementations"
47-
PythonPackagesDir = "python_packages"
48-
ModelsDir = "models"
49-
ConstantsDir = "constants"
50-
ContextsDir = "contexts"
51-
ResourceStatusesDir = "resource_statuses"
52-
WorkloadSpecsDir = "workload_specs"
53-
LogPrefixesDir = "log_prefixes"
39+
AppsDir = "apps"
40+
APIsDir = "apis"
41+
DataDir = "data"
42+
RawDataDir = "data_raw"
43+
TrainingDataDir = "data_training"
44+
AggregatorsDir = "aggregators"
45+
AggregatesDir = "aggregates"
46+
TransformersDir = "transformers"
47+
ModelImplsDir = "model_implementations"
48+
PythonPackagesDir = "python_packages"
49+
ModelsDir = "models"
50+
ConstantsDir = "constants"
51+
ContextsDir = "contexts"
52+
ResourceStatusesDir = "resource_statuses"
53+
WorkloadSpecsDir = "workload_specs"
54+
LogPrefixesDir = "log_prefixes"
55+
RawColumnsDir = "raw_columns"
56+
TransformedColumnsDir = "transformed_columns"
5457

5558
TelemetryURL = "https://telemetry.cortexlabs.dev"
5659
)

‎pkg/lib/configreader/string.go‎

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -28,15 +28,16 @@ import (
2828
)
2929

3030
type StringValidation struct {
31-
Required bool
32-
Default string
33-
AllowEmpty bool
34-
AllowedValues []string
35-
Prefix string
36-
AlphaNumericDashDotUnderscore bool
37-
AlphaNumericDashUnderscore bool
38-
DNS1035 bool
39-
Validator func(string) (string, error)
31+
Required bool
32+
Default string
33+
AllowEmpty bool
34+
AllowedValues []string
35+
Prefix string
36+
AlphaNumericDashDotUnderscoreOrEmpty bool
37+
AlphaNumericDashDotUnderscore bool
38+
AlphaNumericDashUnderscore bool
39+
DNS1035 bool
40+
Validator func(string) (string, error)
4041
}
4142

4243
func EnvVar(envVarName string) string {
@@ -190,6 +191,12 @@ func ValidateStringVal(val string, v *StringValidation) error {
190191
}
191192
}
192193

194+
if v.AlphaNumericDashDotUnderscoreOrEmpty {
195+
if !regex.CheckAlphaNumericDashDotUnderscore(val) && val != "" {
196+
return ErrorAlphaNumericDashDotUnderscore(val)
197+
}
198+
}
199+
193200
if v.DNS1035 {
194201
if err := urls.CheckDNS1035(val); err != nil {
195202
return err

‎pkg/operator/api/context/context.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ type Resource interface {
5555
GetID() string
5656
GetIDWithTags() string
5757
GetResourceFields() *ResourceFields
58+
GetMetadataKey() string
5859
}
5960

6061
type ComputedResource interface {
@@ -72,6 +73,7 @@ type ResourceFields struct {
7273
ID string `json:"id"`
7374
IDWithTags string `json:"id_with_tags"`
7475
ResourceType resource.Type `json:"resource_type"`
76+
MetadataKey string `json:"metadata_key"`
7577
}
7678

7779
type ComputedResourceFields struct {
@@ -91,6 +93,10 @@ func (r *ResourceFields) GetResourceFields() *ResourceFields {
9193
return r
9294
}
9395

96+
func (r *ResourceFields) GetMetadataKey() string {
97+
return r.MetadataKey
98+
}
99+
94100
func (r *ComputedResourceFields) GetWorkloadID() string {
95101
return r.WorkloadID
96102
}

‎pkg/operator/api/context/models.go‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,11 @@ type Model struct {
3838
}
3939

4040
type TrainingDataset struct {
41-
userconfig.ResourceConfigFields
41+
userconfig.ResourceFields
4242
*ComputedResourceFields
43-
ModelName string `json:"model_name"`
44-
TrainKey string `json:"train_key"`
45-
EvalKey string `json:"eval_key"`
46-
MetadataKey string `json:"metadata_key"`
43+
ModelName string `json:"model_name"`
44+
TrainKey string `json:"train_key"`
45+
EvalKey string `json:"eval_key"`
4746
}
4847

4948
func (trainingDataset *TrainingDataset) GetResourceType() resource.Type {

‎pkg/operator/api/context/python_packages.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ import (
2424
type PythonPackages map[string]*PythonPackage
2525

2626
type PythonPackage struct {
27-
userconfig.ResourceConfigFields
27+
userconfig.ResourceFields
2828
*ComputedResourceFields
2929
SrcKey string `json:"src_key"`
3030
PackageKey string `json:"package_key"`

‎pkg/operator/api/userconfig/aggregates.go‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,12 @@ import (
2626
type Aggregates []*Aggregate
2727

2828
type Aggregate struct {
29-
ResourceConfigFields
30-
Aggregator string `json:"aggregator" yaml:"aggregator"`
31-
Inputs *Inputs `json:"inputs" yaml:"inputs"`
32-
Compute *SparkCompute `json:"compute" yaml:"compute"`
33-
Tags Tags `json:"tags" yaml:"tags"`
29+
ResourceFields
30+
Aggregator string `json:"aggregator" yaml:"aggregator"`
31+
AggregatorPath *string `json:"aggregator_path" yaml:"aggregator_path"`
32+
Inputs *Inputs `json:"inputs" yaml:"inputs"`
33+
Compute *SparkCompute `json:"compute" yaml:"compute"`
34+
Tags Tags `json:"tags" yaml:"tags"`
3435
}
3536

3637
var aggregateValidation = &configreader.StructValidation{
@@ -45,10 +46,14 @@ var aggregateValidation = &configreader.StructValidation{
4546
{
4647
StructField: "Aggregator",
4748
StringValidation: &configreader.StringValidation{
48-
Required: true,
49-
AlphaNumericDashDotUnderscore: true,
49+
AllowEmpty: true,
50+
AlphaNumericDashDotUnderscoreOrEmpty: true,
5051
},
5152
},
53+
{
54+
StructField: "AggregatorPath",
55+
StringPtrValidation: &configreader.StringPtrValidation{},
56+
},
5257
inputValuesFieldValidation,
5358
sparkComputeFieldValidation("Compute"),
5459
tagsFieldValidation,

0 commit comments

Comments
 (0)