remove inferenceOjective from cache

Signed-off-by: Lionel Villard <villard@us.ibm.com>

Author: lionelvillard

pkg/activator/server/controller_manager.go

@@ -56,11 +56,6 @@ func defaultManagerOptions(gknn common.GKNN, metricsServerOptions metricsserver.
 						gknn.Namespace: {},
 					},
 				},
-				&v1alpha2.InferenceObjective{}: {
-					Namespaces: map[string]cache.Config{
-						gknn.Namespace: {},
-					},
-				},
 			},
 		},
 		Metrics: metricsServerOptions,