Merge branch 'master' of github.com:tensorflow/swift-apis

Author: rxwei

Sources/DeepLearning/Layer.swift

@@ -102,6 +102,80 @@ public extension Layer {
     }
 }
 
+/// Adds helpers for standard feed-forward, sequential models.
+public extension Differentiable {
+    @differentiable(wrt: (self, l1, l2))
+    func sequenced<L1: Layer, L2: Layer>(
+        in context: Context, through l1: L1, _ l2: L2)
+        -> L2.Output
+            where L1.Input == Self,
+                  L1.Output == L2.Input {
+        let o1 = l1.applied(to: self, in: context)
+        return l2.applied(to: o1, in: context)
+    }
+
+    @differentiable(wrt: (self, l1, l2, l3))
+    func sequenced<L1: Layer, L2: Layer, L3: Layer>(
+        in context: Context, through l1: L1, _ l2: L2, _ l3: L3)
+        -> L3.Output
+            where L1.Input == Self,
+                  L1.Output == L2.Input,
+                  L2.Output == L3.Input {
+        let o1 = l1.applied(to: self, in: context)
+        let o2 = l2.applied(to: o1, in: context)
+        return l3.applied(to: o2, in: context)
+    }
+
+    @differentiable(wrt: (self, l1, l2, l3, l4))
+    func sequenced<L1: Layer, L2: Layer, L3: Layer, L4: Layer>(
+        in context: Context, through l1: L1, _ l2: L2, _ l3: L3, _ l4: L4)
+        -> L4.Output
+            where L1.Input == Self,
+                  L1.Output == L2.Input,
+                  L2.Output == L3.Input,
+                  L3.Output == L4.Input {
+        let o1 = l1.applied(to: self, in: context)
+        let o2 = l2.applied(to: o1, in: context)
+        let o3 = l3.applied(to: o2, in: context)
+        return l4.applied(to: o3, in: context)
+    }
+
+    @differentiable(wrt: (self, l1, l2, l3, l4, l5))
+    func sequenced<L1: Layer, L2: Layer, L3: Layer, L4: Layer, L5: Layer>(
+        in context: Context, through l1: L1, _ l2: L2, _ l3: L3, _ l4: L4, _ l5: L5)
+        -> L5.Output
+            where L1.Input == Self,
+                  L1.Output == L2.Input,
+                  L2.Output == L3.Input,
+                  L3.Output == L4.Input,
+                  L4.Output == L5.Input {
+        let o1 = l1.applied(to: self, in: context)
+        let o2 = l2.applied(to: o1, in: context)
+        let o3 = l3.applied(to: o2, in: context)
+        let o4 = l4.applied(to: o3, in: context)
+        return l5.applied(to: o4, in: context)
+    }
+
+    @differentiable(wrt: (self, l1, l2, l3, l4, l5, l6))
+    func sequenced<L1: Layer, L2: Layer, L3: Layer, L4: Layer, L5: Layer, L6: Layer>(
+        in context: Context, through l1: L1, _ l2: L2, _ l3: L3, _ l4: L4, _ l5: L5, _ l6: L6)
+        -> L6.Output
+            where L1.Input == Self,
+                  L1.Output == L2.Input,
+                  L2.Output == L3.Input,
+                  L3.Output == L4.Input,
+                  L4.Output == L5.Input,
+                  L5.Output == L6.Input {
+        let o1 = l1.applied(to: self, in: context)
+        let o2 = l2.applied(to: o1, in: context)
+        let o3 = l3.applied(to: o2, in: context)
+        let o4 = l4.applied(to: o3, in: context)
+        let o5 = l5.applied(to: o4, in: context)
+        return l6.applied(to: o5, in: context)
+    }
+}
+
+
 /// A mutable, shareable, owning reference to a tensor.
 public final class Parameter<Scalar: TensorFlowScalar> {
     public var value: Tensor<Scalar>
@@ -137,7 +211,7 @@ public extension Dense where Scalar.RawSignificand: FixedWidthInteger {
     init<G: RandomNumberGenerator>(
         inputSize: Int,
         outputSize: Int,
-        activation: @escaping Activation,
+        activation: @escaping Activation = identity,
         generator: inout G
     ) {
         self.init(weight: Tensor(glorotUniform: [Int32(inputSize), Int32(outputSize)],
@@ -146,7 +220,7 @@ public extension Dense where Scalar.RawSignificand: FixedWidthInteger {
                   activation: activation)
     }
 
-    init(inputSize: Int, outputSize: Int, activation: @escaping Activation) {
+    init(inputSize: Int, outputSize: Int, activation: @escaping Activation = identity) {
       self.init(inputSize: inputSize, outputSize: outputSize, activation: activation,
                 generator: &PhiloxRandomNumberGenerator.global)
     }

Sources/DeepLearning/Loss.swift

@@ -16,16 +16,40 @@
 import TensorFlow
 #endif
 
+/// Computes the mean squared error between logits and labels.
+///
+/// - Parameters:
+///   - logits: One-hot encoded outputs from a neural network.
+///   - labels: One-hot encoded values that correspond to the correct output.
 @differentiable
 public func meanSquaredError<Scalar: TensorFlowFloatingPoint>(
     predicted: Tensor<Scalar>, expected: Tensor<Scalar>
 ) -> Tensor<Scalar> {
     return (expected - predicted).squared().mean()
 }
 
+/// Computes the softmax cross entropy (categorical cross entropy) between logits and labels.
+///
+/// - Parameters:
+///   - logits: One-hot encoded outputs from a neural network.
+///   - labels: One-hot encoded values that correspond to the correct output.
 @differentiable
 public func softmaxCrossEntropy<Scalar: TensorFlowFloatingPoint>(
     logits: Tensor<Scalar>, labels: Tensor<Scalar>
 ) -> Tensor<Scalar> {
     return -(labels * logSoftmax(logits)).mean(alongAxes: 0).sum()
 }
+
+/// Computes the sigmoid cross entropy (binary cross entropy) between logits and labels.
+///
+/// - Parameters:
+///   - logits: Single continuous values from `0` to `1`.
+///   - labels: Integer values that correspond to the correct output.
+@differentiable
+public func sigmoidCrossEntropy<Scalar: TensorFlowFloatingPoint>(
+    logits: Tensor<Scalar>, labels: Tensor<Scalar>
+) -> Tensor<Scalar> {
+    let loss = labels * log(logits) +
+        (Tensor<Scalar>(1) - labels) * log(Tensor<Scalar>(1) - logits)
+    return -loss.mean(alongAxes: 0).sum()
+}

Tests/DeepLearningTests/SequentialTests.swift

@@ -0,0 +1,47 @@
+// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import XCTest
+@testable import DeepLearning
+
+final class SequentialTests: XCTestCase {
+    func testSequential() {
+        struct Model: Layer {
+            var dense1 = Dense<Float>(inputSize: 2, outputSize: 4, activation: relu)
+            var dense2 = Dense<Float>(inputSize: 4, outputSize: 1, activation: relu)
+
+            @differentiable(wrt: (self, input))
+            func applied(to input: Tensor<Float>, in context: Context) -> Tensor<Float> {
+              return input.sequenced(in: context, through: dense1, dense2)
+            }
+        }
+        var model = Model()
+        let optimizer = SGD(learningRate: 0.02, modelType: type(of: model), scalarType: Float.self)
+        let x: Tensor<Float> = [[0, 0], [0, 1], [1, 0], [1, 1]]
+        let y: Tensor<Float> = [0, 1, 1, 0]
+        let context = Context(learningPhase: .training)
+        for _ in 0..<1000 {
+            let 𝛁model = model.gradient { model -> Tensor<Float> in
+                let ŷ = model.applied(to: x, in: context)
+                return meanSquaredError(predicted: ŷ, expected: y)
+            }
+            optimizer.update(&model.allDifferentiableVariables, along: 𝛁model)
+        }
+        print(model.inferring(from: [[0, 0], [0, 1], [1, 0], [1, 1]]))
+    }
+
+    static var allTests = [
+        ("testSequential", testSequential)
+    ]
+}

Tests/DeepLearningTests/XCTestManifests.swift

@@ -19,6 +19,7 @@ public func allTests() -> [XCTestCaseEntry] {
     return [
         testCase(PRNGTests.allTests),
         testCase(TrivialModelTests.allTests),
+        testCase(SequentialTests.allTests),
     ]
 }
 #endif