From 33707231eeb3af63b26a7c865f2283e864efc9d3 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sun, 13 Nov 2022 16:23:23 -0600 Subject: [PATCH 01/52] Define Keras interface in core project (WIP). --- src/SciSharp.TensorFlow.Redist/README.md | 4 +- src/TensorFlowNET.Console/Program.cs | 3 + src/TensorFlowNET.Console/SimpleRnnTest.cs | 31 ++++ .../Tensorflow.Console.csproj | 4 +- .../Keras/ArgsDefinition/LSTMArgs.cs | 22 --- .../Keras/ArgsDefinition/Lstm/LSTMArgs.cs | 12 ++ .../ArgsDefinition/{ => Lstm}/LSTMCellArgs.cs | 2 +- .../Keras/ArgsDefinition/RNNArgs.cs | 21 --- .../Keras/ArgsDefinition/Rnn/RNNArgs.cs | 45 +++++ .../Keras/ArgsDefinition/Rnn/SimpleRNNArgs.cs | 7 + .../{ => Rnn}/StackedRNNCellsArgs.cs | 2 +- .../Keras/ArgsDefinition/SimpleRNNArgs.cs | 30 ---- src/TensorFlowNET.Core/Keras/IKerasApi.cs | 12 ++ .../Keras/IPreprocessing.cs | 16 ++ .../Keras/Layers/ILayersApi.Activation.cs | 20 +++ .../Keras/Layers/ILayersApi.Attention.cs | 28 +++ .../Keras/Layers/ILayersApi.Cropping.cs | 13 ++ .../Keras/Layers/ILayersApi.Merging.cs | 10 ++ .../Keras/Layers/ILayersApi.Reshaping.cs | 18 ++ .../Keras/Layers/ILayersApi.cs | 169 ++++++++++++++++++ .../NumPy/Implementation/RandomizedImpl.cs | 10 +- .../Operations/NnOps/RNNCell.cs | 1 + .../Tensorflow.Binding.csproj | 28 ++- src/TensorFlowNET.Core/tensorflow.cs | 3 + src/TensorFlowNET.Keras/KerasApi.cs | 3 + src/TensorFlowNET.Keras/KerasInterface.cs | 5 +- .../Layers/LayersApi.Activation.cs | 18 +- .../Layers/LayersApi.Attention.cs | 4 +- .../Layers/LayersApi.Cropping.cs | 6 +- .../Layers/LayersApi.Merging.cs | 2 +- .../Layers/LayersApi.Reshaping.cs | 16 +- src/TensorFlowNET.Keras/Layers/LayersApi.cs | 98 +++++----- .../Layers/{ => Lstm}/LSTM.cs | 5 +- .../Layers/{ => Lstm}/LSTMCell.cs | 4 +- .../Layers/{ => Rnn}/RNN.cs | 6 +- .../Layers/Rnn/SimpleRNN.cs | 31 ++++ .../Layers/Rnn/SimpleRNNCell.cs | 21 +++ .../Layers/{ => Rnn}/StackedRNNCells.cs | 3 +- src/TensorFlowNET.Keras/Layers/SimpleRNN.cs | 14 -- .../Preprocessings/Preprocessing.Resizing.cs | 2 +- .../Preprocessings/Preprocessing.cs | 4 +- .../Tensorflow.Keras.csproj | 18 +- src/TensorFlowNET.Keras/tf.layers.cs | 2 +- src/python/.vscode/launch.json | 16 ++ src/python/simple_rnn.py | 15 ++ .../Layers/AttentionTest.cs | 4 +- .../Layers/LayersTest.cs | 3 +- .../Tensorflow.Keras.UnitTest.csproj | 2 +- .../Tensorflow.Binding.UnitTest.csproj | 2 +- 49 files changed, 610 insertions(+), 205 deletions(-) create mode 100644 src/TensorFlowNET.Console/SimpleRnnTest.cs delete mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/LSTMArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Lstm/LSTMArgs.cs rename src/TensorFlowNET.Core/Keras/ArgsDefinition/{ => Lstm}/LSTMCellArgs.cs (53%) delete mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/RNNArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNArgs.cs rename src/TensorFlowNET.Core/Keras/ArgsDefinition/{ => Rnn}/StackedRNNCellsArgs.cs (82%) delete mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/SimpleRNNArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/IKerasApi.cs create mode 100644 src/TensorFlowNET.Core/Keras/IPreprocessing.cs create mode 100644 src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Activation.cs create mode 100644 src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Attention.cs create mode 100644 src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Cropping.cs create mode 100644 src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Merging.cs create mode 100644 src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Reshaping.cs create mode 100644 src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs rename src/TensorFlowNET.Keras/Layers/{ => Lstm}/LSTM.cs (87%) rename src/TensorFlowNET.Keras/Layers/{ => Lstm}/LSTMCell.cs (72%) rename src/TensorFlowNET.Keras/Layers/{ => Rnn}/RNN.cs (95%) create mode 100644 src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs create mode 100644 src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs rename src/TensorFlowNET.Keras/Layers/{ => Rnn}/StackedRNNCells.cs (98%) delete mode 100644 src/TensorFlowNET.Keras/Layers/SimpleRNN.cs create mode 100644 src/python/.vscode/launch.json create mode 100644 src/python/simple_rnn.py diff --git a/src/SciSharp.TensorFlow.Redist/README.md b/src/SciSharp.TensorFlow.Redist/README.md index 141bba352..4002aa21d 100644 --- a/src/SciSharp.TensorFlow.Redist/README.md +++ b/src/SciSharp.TensorFlow.Redist/README.md @@ -26,7 +26,7 @@ Related merged [commits](https://github.com/SciSharp/TensorFlow.NET/commit/854a5 #### Download pre-build package -[Mac OSX CPU](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-darwin-x86_64-2.4.0.tar.gz), [Linux CPU](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-linux-x86_64-2.4.0.tar.gz), [Linux GPU](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-gpu-linux-x86_64-2.4.0.tar.gz), [Windows CPU](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-windows-x86_64-2.4.0.tar.gz), [Windows GPU](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-gpu-windows-x86_64-2.4.0.zip) +[Mac OSX CPU](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-darwin-x86_64-2.10.0.tar.gz), [Linux CPU](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-linux-x86_64-2.10.0.tar.gz), [Linux GPU](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-gpu-linux-x86_64-2.10.0.tar.gz), [Windows CPU](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-windows-x86_64-2.10.0.zip), [Windows GPU](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-gpu-windows-x86_64-2.10.0.zip) @@ -35,6 +35,6 @@ Related merged [commits](https://github.com/SciSharp/TensorFlow.NET/commit/854a5 On Windows, the tar command does not support extracting archives with symlinks. So when `dotnet pack` runs on Windows it will only package the Windows binaries. 1. Run `dotnet pack SciSharp.TensorFlow.Redist.nupkgproj` under `src/SciSharp.TensorFlow.Redist` directory in Linux. -2. Run `dotnet nuget push SciSharp.TensorFlow.Redist.2.4.0.nupkg -k APIKEY -s https://api.nuget.org/v3/index.json -t 600` +2. Run `dotnet nuget push SciSharp.TensorFlow.Redist.2.10.0.nupkg -k APIKEY -s https://api.nuget.org/v3/index.json -t 600` diff --git a/src/TensorFlowNET.Console/Program.cs b/src/TensorFlowNET.Console/Program.cs index 4b7f52deb..638fe0a3e 100644 --- a/src/TensorFlowNET.Console/Program.cs +++ b/src/TensorFlowNET.Console/Program.cs @@ -10,6 +10,9 @@ static void Main(string[] args) var diag = new Diagnostician(); // diag.Diagnose(@"D:\memory.txt"); + var rnn = new SimpleRnnTest(); + rnn.Run(); + // this class is used explor new features. var exploring = new Exploring(); // exploring.Run(); diff --git a/src/TensorFlowNET.Console/SimpleRnnTest.cs b/src/TensorFlowNET.Console/SimpleRnnTest.cs new file mode 100644 index 000000000..b61cee9c8 --- /dev/null +++ b/src/TensorFlowNET.Console/SimpleRnnTest.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Keras; +using Tensorflow.NumPy; +using static Tensorflow.Binding; +using static Tensorflow.KerasApi; + +namespace Tensorflow +{ + public class SimpleRnnTest + { + public void Run() + { + tf.keras = new KerasInterface(); + var inputs = np.random.random((32, 10, 8)).astype(np.float32); + var simple_rnn = tf.keras.layers.SimpleRNN(4); + var output = simple_rnn.Apply(inputs); // The output has shape `[32, 4]`. + if (output.shape == (32, 4)) + { + + } + /*simple_rnn = tf.keras.layers.SimpleRNN( + 4, return_sequences = True, return_state = True) + + # whole_sequence_output has shape `[32, 10, 4]`. + # final_state has shape `[32, 4]`. + whole_sequence_output, final_state = simple_rnn(inputs)*/ + } + } +} diff --git a/src/TensorFlowNET.Console/Tensorflow.Console.csproj b/src/TensorFlowNET.Console/Tensorflow.Console.csproj index 058722eb8..e66c7033c 100644 --- a/src/TensorFlowNET.Console/Tensorflow.Console.csproj +++ b/src/TensorFlowNET.Console/Tensorflow.Console.csproj @@ -6,7 +6,7 @@ Tensorflow Tensorflow AnyCPU;x64 - 9.0 + 11.0 @@ -20,7 +20,7 @@ - + diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/LSTMArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/LSTMArgs.cs deleted file mode 100644 index 0a2555a69..000000000 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/LSTMArgs.cs +++ /dev/null @@ -1,22 +0,0 @@ -namespace Tensorflow.Keras.ArgsDefinition -{ - public class LSTMArgs : RNNArgs - { - public int Units { get; set; } - public Activation Activation { get; set; } - public Activation RecurrentActivation { get; set; } - public IInitializer KernelInitializer { get; set; } - public IInitializer RecurrentInitializer { get; set; } - public IInitializer BiasInitializer { get; set; } - public bool UnitForgetBias { get; set; } - public float Dropout { get; set; } - public float RecurrentDropout { get; set; } - public int Implementation { get; set; } - public bool ReturnSequences { get; set; } - public bool ReturnState { get; set; } - public bool GoBackwards { get; set; } - public bool Stateful { get; set; } - public bool TimeMajor { get; set; } - public bool Unroll { get; set; } - } -} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Lstm/LSTMArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Lstm/LSTMArgs.cs new file mode 100644 index 000000000..b08d21d88 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Lstm/LSTMArgs.cs @@ -0,0 +1,12 @@ +using Tensorflow.Keras.ArgsDefinition.Rnn; + +namespace Tensorflow.Keras.ArgsDefinition.Lstm +{ + public class LSTMArgs : RNNArgs + { + public bool UnitForgetBias { get; set; } + public float Dropout { get; set; } + public float RecurrentDropout { get; set; } + public int Implementation { get; set; } + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/LSTMCellArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Lstm/LSTMCellArgs.cs similarity index 53% rename from src/TensorFlowNET.Core/Keras/ArgsDefinition/LSTMCellArgs.cs rename to src/TensorFlowNET.Core/Keras/ArgsDefinition/Lstm/LSTMCellArgs.cs index 62f9a0c4e..fb0868dc5 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/LSTMCellArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Lstm/LSTMCellArgs.cs @@ -1,4 +1,4 @@ -namespace Tensorflow.Keras.ArgsDefinition +namespace Tensorflow.Keras.ArgsDefinition.Lstm { public class LSTMCellArgs : LayerArgs { diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/RNNArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/RNNArgs.cs deleted file mode 100644 index 3ebcf617a..000000000 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/RNNArgs.cs +++ /dev/null @@ -1,21 +0,0 @@ -using System.Collections.Generic; - -namespace Tensorflow.Keras.ArgsDefinition -{ - public class RNNArgs : LayerArgs - { - public interface IRnnArgCell : ILayer - { - object state_size { get; } - } - - public IRnnArgCell Cell { get; set; } = null; - public bool ReturnSequences { get; set; } = false; - public bool ReturnState { get; set; } = false; - public bool GoBackwards { get; set; } = false; - public bool Stateful { get; set; } = false; - public bool Unroll { get; set; } = false; - public bool TimeMajor { get; set; } = false; - public Dictionary Kwargs { get; set; } = null; - } -} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs new file mode 100644 index 000000000..da5279257 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs @@ -0,0 +1,45 @@ +using System.Collections.Generic; + +namespace Tensorflow.Keras.ArgsDefinition.Rnn +{ + public class RNNArgs : LayerArgs + { + public interface IRnnArgCell : ILayer + { + object state_size { get; } + } + + public IRnnArgCell Cell { get; set; } = null; + public bool ReturnSequences { get; set; } = false; + public bool ReturnState { get; set; } = false; + public bool GoBackwards { get; set; } = false; + public bool Stateful { get; set; } = false; + public bool Unroll { get; set; } = false; + public bool TimeMajor { get; set; } = false; + public Dictionary Kwargs { get; set; } = null; + + public int Units { get; set; } + public Activation Activation { get; set; } + public Activation RecurrentActivation { get; set; } + public bool UseBias { get; set; } = true; + public IInitializer KernelInitializer { get; set; } + public IInitializer RecurrentInitializer { get; set; } + public IInitializer BiasInitializer { get; set; } + + // kernel_regularizer=None, + // recurrent_regularizer=None, + // bias_regularizer=None, + // activity_regularizer=None, + // kernel_constraint=None, + // recurrent_constraint=None, + // bias_constraint=None, + // dropout=0., + // recurrent_dropout=0., + // return_sequences=False, + // return_state=False, + // go_backwards=False, + // stateful=False, + // unroll=False, + // **kwargs): + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNArgs.cs new file mode 100644 index 000000000..fcfd694d1 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/SimpleRNNArgs.cs @@ -0,0 +1,7 @@ +namespace Tensorflow.Keras.ArgsDefinition.Rnn +{ + public class SimpleRNNArgs : RNNArgs + { + + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/StackedRNNCellsArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/StackedRNNCellsArgs.cs similarity index 82% rename from src/TensorFlowNET.Core/Keras/ArgsDefinition/StackedRNNCellsArgs.cs rename to src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/StackedRNNCellsArgs.cs index 9b910e17e..fdfadab85 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/StackedRNNCellsArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/StackedRNNCellsArgs.cs @@ -1,6 +1,6 @@ using System.Collections.Generic; -namespace Tensorflow.Keras.ArgsDefinition +namespace Tensorflow.Keras.ArgsDefinition.Rnn { public class StackedRNNCellsArgs : LayerArgs { diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/SimpleRNNArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/SimpleRNNArgs.cs deleted file mode 100644 index 658155875..000000000 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/SimpleRNNArgs.cs +++ /dev/null @@ -1,30 +0,0 @@ -namespace Tensorflow.Keras.ArgsDefinition -{ - public class SimpleRNNArgs : RNNArgs - { - public int Units { get; set; } - public Activation Activation { get; set; } - - // units, - // activation='tanh', - // use_bias=True, - // kernel_initializer='glorot_uniform', - // recurrent_initializer='orthogonal', - // bias_initializer='zeros', - // kernel_regularizer=None, - // recurrent_regularizer=None, - // bias_regularizer=None, - // activity_regularizer=None, - // kernel_constraint=None, - // recurrent_constraint=None, - // bias_constraint=None, - // dropout=0., - // recurrent_dropout=0., - // return_sequences=False, - // return_state=False, - // go_backwards=False, - // stateful=False, - // unroll=False, - // **kwargs): - } -} diff --git a/src/TensorFlowNET.Core/Keras/IKerasApi.cs b/src/TensorFlowNET.Core/Keras/IKerasApi.cs new file mode 100644 index 000000000..660dcbde7 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/IKerasApi.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Keras.Layers; + +namespace Tensorflow.Keras +{ + public interface IKerasApi + { + public ILayersApi layers { get; } + } +} diff --git a/src/TensorFlowNET.Core/Keras/IPreprocessing.cs b/src/TensorFlowNET.Core/Keras/IPreprocessing.cs new file mode 100644 index 000000000..28eea0f56 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/IPreprocessing.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras +{ + public interface IPreprocessing + { + public ILayer Resizing(int height, int width, string interpolation = "bilinear"); + public ILayer TextVectorization(Func standardize = null, + string split = "whitespace", + int max_tokens = -1, + string output_mode = "int", + int output_sequence_length = -1); + } +} diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Activation.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Activation.cs new file mode 100644 index 000000000..73a6787c3 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Activation.cs @@ -0,0 +1,20 @@ +using System; +using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.NumPy; +using Tensorflow.Operations.Activation; + +namespace Tensorflow.Keras.Layers +{ + public partial interface ILayersApi + { + public ILayer ELU(float alpha = 0.1f); + public ILayer SELU(); + public ILayer Softmax(Axis axis); + public ILayer Softplus(); + public ILayer HardSigmoid(); + public ILayer Softsign(); + public ILayer Swish(); + public ILayer Tanh(); + public ILayer Exponential(); + } +} diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Attention.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Attention.cs new file mode 100644 index 000000000..22fb50d3d --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Attention.cs @@ -0,0 +1,28 @@ +using System; +using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.NumPy; + +namespace Tensorflow.Keras.Layers +{ + public partial interface ILayersApi + { + public ILayer Attention(bool use_scale = false, + string score_mode = "dot", + bool causal = false, + float dropout = 0f); + public ILayer MultiHeadAttention(int num_heads, + int key_dim, + int? value_dim = null, + float dropout = 0f, + bool use_bias = true, + Shape output_shape = null, + Shape attention_axes = null, + IInitializer kernel_initializer = null, + IInitializer bias_initializer = null, + IRegularizer kernel_regularizer = null, + IRegularizer bias_regularizer = null, + IRegularizer activity_regularizer = null, + Action kernel_constraint = null, + Action bias_constraint = null); + } +} diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Cropping.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Cropping.cs new file mode 100644 index 000000000..602e7a880 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Cropping.cs @@ -0,0 +1,13 @@ +using System; +using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.NumPy; + +namespace Tensorflow.Keras.Layers +{ + public partial interface ILayersApi + { + public ILayer Cropping1D(NDArray cropping); + public ILayer Cropping2D(NDArray cropping, Cropping2DArgs.DataFormat data_format = Cropping2DArgs.DataFormat.channels_last); + public ILayer Cropping3D(NDArray cropping, Cropping3DArgs.DataFormat data_format = Cropping3DArgs.DataFormat.channels_last); + } +} diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Merging.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Merging.cs new file mode 100644 index 000000000..d0a7f09fd --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Merging.cs @@ -0,0 +1,10 @@ +using System; +using Tensorflow.NumPy; + +namespace Tensorflow.Keras.Layers +{ + public partial interface ILayersApi + { + public ILayer Concatenate(int axis = -1); + } +} diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Reshaping.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Reshaping.cs new file mode 100644 index 000000000..d41e06887 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Reshaping.cs @@ -0,0 +1,18 @@ +using System; +using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.NumPy; + +namespace Tensorflow.Keras.Layers +{ + public partial interface ILayersApi + { + public ILayer Reshape(Shape target_shape); + public ILayer Reshape(object[] target_shape); + + public ILayer UpSampling2D(Shape size = null, + string data_format = null, + string interpolation = "nearest"); + + public ILayer ZeroPadding2D(NDArray padding); + } +} diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs new file mode 100644 index 000000000..5945bb551 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs @@ -0,0 +1,169 @@ +using System; +using static Google.Protobuf.Reflection.FieldDescriptorProto.Types; + +namespace Tensorflow.Keras.Layers +{ + public partial interface ILayersApi + { + public IPreprocessing preprocessing { get; } + + public ILayer Add(); + + public ILayer AveragePooling2D(Shape pool_size = null, + Shape strides = null, + string padding = "valid", + string data_format = null); + + public ILayer BatchNormalization(int axis = -1, + float momentum = 0.99f, + float epsilon = 0.001f, + bool center = true, + bool scale = true, + IInitializer beta_initializer = null, + IInitializer gamma_initializer = null, + IInitializer moving_mean_initializer = null, + IInitializer moving_variance_initializer = null, + bool trainable = true, + string name = null, + bool renorm = false, + float renorm_momentum = 0.99f); + + public ILayer Conv1D(int filters, + Shape kernel_size, + int strides = 1, + string padding = "valid", + string data_format = "channels_last", + int dilation_rate = 1, + int groups = 1, + string activation = null, + bool use_bias = true, + string kernel_initializer = "glorot_uniform", + string bias_initializer = "zeros"); + + public ILayer Conv2D(int filters, + Shape kernel_size = null, + Shape strides = null, + string padding = "valid", + string data_format = null, + Shape dilation_rate = null, + int groups = 1, + Activation activation = null, + bool use_bias = true, + IInitializer kernel_initializer = null, + IInitializer bias_initializer = null, + IRegularizer kernel_regularizer = null, + IRegularizer bias_regularizer = null, + IRegularizer activity_regularizer = null); + + public ILayer Conv2D(int filters, + Shape kernel_size = null, + Shape strides = null, + string padding = "valid", + string data_format = null, + Shape dilation_rate = null, + int groups = 1, + string activation = null, + bool use_bias = true, + string kernel_initializer = "glorot_uniform", + string bias_initializer = "zeros"); + + public ILayer Dense(int units); + public ILayer Dense(int units, + string activation = null, + Shape input_shape = null); + public ILayer Dense(int units, + Activation activation = null, + IInitializer kernel_initializer = null, + bool use_bias = true, + IInitializer bias_initializer = null, + Shape input_shape = null); + + public ILayer Dropout(float rate, Shape noise_shape = null, int? seed = null); + + public ILayer Embedding(int input_dim, + int output_dim, + IInitializer embeddings_initializer = null, + bool mask_zero = false, + Shape input_shape = null, + int input_length = -1); + + public ILayer EinsumDense(string equation, + Shape output_shape, + string bias_axes, + Activation activation = null, + IInitializer kernel_initializer = null, + IInitializer bias_initializer = null, + IRegularizer kernel_regularizer = null, + IRegularizer bias_regularizer = null, + IRegularizer activity_regularizer = null, + Action kernel_constraint = null, + Action bias_constraint = null); + + public ILayer Flatten(string data_format = null); + + public ILayer GlobalAveragePooling1D(string data_format = "channels_last"); + public ILayer GlobalAveragePooling2D(); + public ILayer GlobalAveragePooling2D(string data_format = "channels_last"); + public ILayer GlobalMaxPooling1D(string data_format = "channels_last"); + public ILayer GlobalMaxPooling2D(string data_format = "channels_last"); + + public Tensors Input(Shape shape, + string name = null, + bool sparse = false, + bool ragged = false); + public ILayer InputLayer(Shape input_shape, + string name = null, + bool sparse = false, + bool ragged = false); + + public ILayer LayerNormalization(Axis? axis, + float epsilon = 1e-3f, + bool center = true, + bool scale = true, + IInitializer beta_initializer = null, + IInitializer gamma_initializer = null); + + public ILayer LeakyReLU(float alpha = 0.3f); + + public ILayer LSTM(int units, + Activation activation = null, + Activation recurrent_activation = null, + bool use_bias = true, + IInitializer kernel_initializer = null, + IInitializer recurrent_initializer = null, + IInitializer bias_initializer = null, + bool unit_forget_bias = true, + float dropout = 0f, + float recurrent_dropout = 0f, + int implementation = 2, + bool return_sequences = false, + bool return_state = false, + bool go_backwards = false, + bool stateful = false, + bool time_major = false, + bool unroll = false); + + public ILayer MaxPooling1D(int? pool_size = null, + int? strides = null, + string padding = "valid", + string data_format = null); + public ILayer MaxPooling2D(Shape pool_size = null, + Shape strides = null, + string padding = "valid", + string data_format = null); + + public ILayer Permute(int[] dims); + + public ILayer Rescaling(float scale, + float offset = 0, + Shape input_shape = null); + + public ILayer SimpleRNN(int units, + string activation = "tanh", + string kernel_initializer = "glorot_uniform", + string recurrent_initializer = "orthogonal", + string bias_initializer = "zeros"); + + public ILayer Subtract(); + } +} diff --git a/src/TensorFlowNET.Core/NumPy/Implementation/RandomizedImpl.cs b/src/TensorFlowNET.Core/NumPy/Implementation/RandomizedImpl.cs index 222b10bb0..064c7362f 100644 --- a/src/TensorFlowNET.Core/NumPy/Implementation/RandomizedImpl.cs +++ b/src/TensorFlowNET.Core/NumPy/Implementation/RandomizedImpl.cs @@ -20,11 +20,11 @@ public void shuffle(NDArray x) Marshal.Copy(y.BufferToArray(), 0, x.TensorDataPointer, (int)x.bytesize); } - public NDArray rand(params int[] shape) - => throw new NotImplementedException(""); + public NDArray random(Shape size) + => uniform(low: 0, high: 1, size: size); [AutoNumPy] - public NDArray randint(int low, int? high = null, Shape size = null, TF_DataType dtype = TF_DataType.TF_INT32) + public NDArray randint(int low, int? high = null, Shape? size = null, TF_DataType dtype = TF_DataType.TF_INT32) { if(high == null) { @@ -41,11 +41,11 @@ public NDArray randn(params int[] shape) => new NDArray(random_ops.random_normal(shape ?? Shape.Scalar)); [AutoNumPy] - public NDArray normal(float loc = 0.0f, float scale = 1.0f, Shape size = null) + public NDArray normal(float loc = 0.0f, float scale = 1.0f, Shape? size = null) => new NDArray(random_ops.random_normal(size ?? Shape.Scalar, mean: loc, stddev: scale)); [AutoNumPy] - public NDArray uniform(float low = 0.0f, float high = 1.0f, Shape size = null) + public NDArray uniform(float low = 0.0f, float high = 1.0f, Shape? size = null) => new NDArray(random_ops.random_uniform(size ?? Shape.Scalar, low, high)); } } diff --git a/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs b/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs index 7c5b21b68..041268b70 100644 --- a/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs +++ b/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs @@ -18,6 +18,7 @@ limitations under the License. using System.Collections.Generic; using Tensorflow.Keras; using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; using Tensorflow.Operations; using Tensorflow.Util; diff --git a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj index 4bd0a4908..364498268 100644 --- a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj +++ b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj @@ -5,8 +5,8 @@ Tensorflow.Binding Tensorflow 2.2.0 - 0.70.2 - 9.0 + 0.100.0 + 10.0 enable Haiping Chen, Meinrad Recheis, Eli Belash SciSharp STACK @@ -20,9 +20,9 @@ Google's TensorFlow full binding in .NET Standard. Building, training and infering deep learning models. https://tensorflownet.readthedocs.io - 0.70.1.0 + 0.100.0.0 - tf.net 0.70.x and above are based on tensorflow native 2.7.0 + tf.net 0.100.x and above are based on tensorflow native 2.10.0 * Eager Mode is added finally. * tf.keras is partially working. @@ -35,14 +35,17 @@ https://tensorflownet.readthedocs.io tf.net 0.4x.x aligns with TensorFlow v2.4.1 native library. tf.net 0.6x.x aligns with TensorFlow v2.6.x native library. - tf.net 0.7x.x aligns with TensorFlow v2.7.x native library. - 0.70.1.0 + tf.net 0.7x.x aligns with TensorFlow v2.7.x native library. + tf.net 0.10x.x aligns with TensorFlow v2.10.x native library. + + 0.100.0.0 LICENSE true true Open.snk AnyCPU;x64 TensorFlow.NET + Debug;Release;GPU @@ -51,6 +54,12 @@ https://tensorflownet.readthedocs.io AnyCPU + + true + TRACE;DEBUG;TRACK_TENSOR_LIFE_1 + AnyCPU + + true TRACE;DEBUG;TRACK_TENSOR_LIFE1 @@ -58,6 +67,13 @@ https://tensorflownet.readthedocs.io TensorFlow.NET.xml + + true + TRACE;DEBUG;TRACK_TENSOR_LIFE1 + x64 + TensorFlow.NET.xml + + true diff --git a/src/TensorFlowNET.Core/tensorflow.cs b/src/TensorFlowNET.Core/tensorflow.cs index 8a2c78a7e..e02723b7c 100644 --- a/src/TensorFlowNET.Core/tensorflow.cs +++ b/src/TensorFlowNET.Core/tensorflow.cs @@ -20,6 +20,7 @@ limitations under the License. using Tensorflow.Contexts; using Tensorflow.Eager; using Tensorflow.Gradients; +using Tensorflow.Keras; namespace Tensorflow { @@ -51,6 +52,8 @@ public partial class tensorflow ThreadLocal _runner = new ThreadLocal(() => new EagerRunner()); public IEagerRunner Runner => _runner.Value; + public IKerasApi keras { get; set; } + public tensorflow() { Logger = new LoggerConfiguration() diff --git a/src/TensorFlowNET.Keras/KerasApi.cs b/src/TensorFlowNET.Keras/KerasApi.cs index d10ced0cb..f79c2b5f2 100644 --- a/src/TensorFlowNET.Keras/KerasApi.cs +++ b/src/TensorFlowNET.Keras/KerasApi.cs @@ -2,6 +2,9 @@ namespace Tensorflow { + /// + /// Deprecated, will use tf.keras + /// public static class KerasApi { public static KerasInterface keras { get; } = new KerasInterface(); diff --git a/src/TensorFlowNET.Keras/KerasInterface.cs b/src/TensorFlowNET.Keras/KerasInterface.cs index 02362a55e..5bf9f97f3 100644 --- a/src/TensorFlowNET.Keras/KerasInterface.cs +++ b/src/TensorFlowNET.Keras/KerasInterface.cs @@ -10,18 +10,17 @@ using Tensorflow.Keras.Metrics; using Tensorflow.Keras.Models; using Tensorflow.Keras.Optimizers; -using Tensorflow.Keras.Saving; using Tensorflow.Keras.Utils; using System.Threading; namespace Tensorflow.Keras { - public class KerasInterface + public class KerasInterface : IKerasApi { public KerasDataset datasets { get; } = new KerasDataset(); public Initializers initializers { get; } = new Initializers(); public Regularizers regularizers { get; } = new Regularizers(); - public LayersApi layers { get; } = new LayersApi(); + public ILayersApi layers { get; } = new LayersApi(); public LossesApi losses { get; } = new LossesApi(); public Activations activations { get; } = new Activations(); public Preprocessing preprocessing { get; } = new Preprocessing(); diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.Activation.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.Activation.cs index 0978d0d3e..24a568390 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.Activation.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.Activation.cs @@ -7,16 +7,16 @@ namespace Tensorflow.Keras.Layers { public partial class LayersApi { - public ELU ELU ( float alpha = 0.1f ) + public ILayer ELU ( float alpha = 0.1f ) => new ELU(new ELUArgs { Alpha = alpha }); - public SELU SELU () + public ILayer SELU () => new SELU(new LayerArgs { }); - public Softmax Softmax ( Axis axis ) => new Softmax(new SoftmaxArgs { axis = axis }); - public Softplus Softplus () => new Softplus(new LayerArgs { }); - public HardSigmoid HardSigmoid () => new HardSigmoid(new LayerArgs { }); - public Softsign Softsign () => new Softsign(new LayerArgs { }); - public Swish Swish () => new Swish(new LayerArgs { }); - public Tanh Tanh () => new Tanh(new LayerArgs { }); - public Exponential Exponential () => new Exponential(new LayerArgs { }); + public ILayer Softmax ( Axis axis ) => new Softmax(new SoftmaxArgs { axis = axis }); + public ILayer Softplus () => new Softplus(new LayerArgs { }); + public ILayer HardSigmoid () => new HardSigmoid(new LayerArgs { }); + public ILayer Softsign () => new Softsign(new LayerArgs { }); + public ILayer Swish () => new Swish(new LayerArgs { }); + public ILayer Tanh () => new Tanh(new LayerArgs { }); + public ILayer Exponential () => new Exponential(new LayerArgs { }); } } diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.Attention.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.Attention.cs index 5effd1752..859e9c14d 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.Attention.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.Attention.cs @@ -10,7 +10,7 @@ namespace Tensorflow.Keras.Layers { public partial class LayersApi { - public Attention Attention(bool use_scale = false, + public ILayer Attention(bool use_scale = false, string score_mode = "dot", bool causal = false, float dropout = 0f) => @@ -21,7 +21,7 @@ public Attention Attention(bool use_scale = false, causal = causal, dropout = dropout }); - public MultiHeadAttention MultiHeadAttention(int num_heads, + public ILayer MultiHeadAttention(int num_heads, int key_dim, int? value_dim = null, float dropout = 0f, diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.Cropping.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.Cropping.cs index f4d2230cd..339ddb85b 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.Cropping.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.Cropping.cs @@ -10,7 +10,7 @@ public partial class LayersApi { /// Cropping layer for 1D input /// /// cropping size - public Cropping1D Cropping1D ( NDArray cropping ) + public ILayer Cropping1D ( NDArray cropping ) => new Cropping1D(new CroppingArgs { cropping = cropping }); @@ -18,7 +18,7 @@ public Cropping1D Cropping1D ( NDArray cropping ) /// /// Cropping layer for 2D input
/// - public Cropping2D Cropping2D ( NDArray cropping, Cropping2DArgs.DataFormat data_format = Cropping2DArgs.DataFormat.channels_last ) + public ILayer Cropping2D ( NDArray cropping, Cropping2DArgs.DataFormat data_format = Cropping2DArgs.DataFormat.channels_last ) => new Cropping2D(new Cropping2DArgs { cropping = cropping, data_format = data_format @@ -27,7 +27,7 @@ public Cropping2D Cropping2D ( NDArray cropping, Cropping2DArgs.DataFormat data_ /// /// Cropping layer for 3D input
/// - public Cropping3D Cropping3D ( NDArray cropping, Cropping3DArgs.DataFormat data_format = Cropping3DArgs.DataFormat.channels_last ) + public ILayer Cropping3D ( NDArray cropping, Cropping3DArgs.DataFormat data_format = Cropping3DArgs.DataFormat.channels_last ) => new Cropping3D(new Cropping3DArgs { cropping = cropping, data_format = data_format diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.Merging.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.Merging.cs index ecf8c0a63..d94bfb4d8 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.Merging.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.Merging.cs @@ -13,7 +13,7 @@ public partial class LayersApi /// /// Axis along which to concatenate. /// - public Concatenate Concatenate(int axis = -1) + public ILayer Concatenate(int axis = -1) => new Concatenate(new MergeArgs { Axis = axis diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.Reshaping.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.Reshaping.cs index 5cfec89ee..d3db1d663 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.Reshaping.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.Reshaping.cs @@ -11,7 +11,7 @@ public partial class LayersApi { /// /// /// - public ZeroPadding2D ZeroPadding2D ( NDArray padding ) + public ILayer ZeroPadding2D ( NDArray padding ) => new ZeroPadding2D(new ZeroPadding2DArgs { Padding = padding }); @@ -24,7 +24,7 @@ public ZeroPadding2D ZeroPadding2D ( NDArray padding ) /// /// /// - public UpSampling2D UpSampling2D ( Shape size = null, + public ILayer UpSampling2D ( Shape size = null, string data_format = null, string interpolation = "nearest" ) => new UpSampling2D(new UpSampling2DArgs { @@ -34,7 +34,7 @@ public UpSampling2D UpSampling2D ( Shape size = null, /// /// Permutes the dimensions of the input according to a given pattern. /// - public Permute Permute ( int[] dims ) + public ILayer Permute ( int[] dims ) => new Permute(new PermuteArgs { dims = dims }); @@ -44,12 +44,12 @@ public Permute Permute ( int[] dims ) /// /// /// - public Reshape Reshape ( Shape target_shape ) - => new Reshape(new ReshapeArgs { - TargetShape = target_shape - }); + public ILayer Reshape ( Shape target_shape ) + => new Reshape(new ReshapeArgs { + TargetShape = target_shape + }); - public Reshape Reshape ( object[] target_shape ) + public ILayer Reshape ( object[] target_shape ) => new Reshape(new ReshapeArgs { TargetShapeObjects = target_shape }); diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index 48856735c..8498f5ac6 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -1,16 +1,18 @@ using System; -using Tensorflow.NumPy; -using System.Collections.Generic; using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.ArgsDefinition.Lstm; +using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; +using Tensorflow.Keras.Layers.Lstm; +using Tensorflow.Keras.Layers.Rnn; using static Tensorflow.Binding; using static Tensorflow.KerasApi; namespace Tensorflow.Keras.Layers { - public partial class LayersApi + public partial class LayersApi : ILayersApi { - public Preprocessing preprocessing { get; } = new Preprocessing(); + public IPreprocessing preprocessing { get; } = new Preprocessing(); /// /// Layer that normalizes its inputs. @@ -38,7 +40,7 @@ public partial class LayersApi /// Note that momentum is still applied to get the means and variances for inference. /// /// Tensor of the same shape as input. - public BatchNormalization BatchNormalization(int axis = -1, + public ILayer BatchNormalization(int axis = -1, float momentum = 0.99f, float epsilon = 0.001f, bool center = true, @@ -84,7 +86,7 @@ public BatchNormalization BatchNormalization(int axis = -1, /// Initializer for the kernel weights matrix (see keras.initializers). /// Initializer for the bias vector (see keras.initializers). /// A tensor of rank 3 representing activation(conv1d(inputs, kernel) + bias). - public Conv1D Conv1D(int filters, + public ILayer Conv1D(int filters, Shape kernel_size, int strides = 1, string padding = "valid", @@ -131,7 +133,7 @@ public Conv1D Conv1D(int filters, /// Regularizer function applied to the bias vector (see keras.regularizers). /// Regularizer function applied to the output of the layer (its "activation") (see keras.regularizers). /// A tensor of rank 4+ representing activation(conv2d(inputs, kernel) + bias). - public Conv2D Conv2D(int filters, + public ILayer Conv2D(int filters, Shape kernel_size = null, Shape strides = null, string padding = "valid", @@ -184,7 +186,7 @@ public Conv2D Conv2D(int filters, /// The name of the regularizer function applied to the bias vector (see keras.regularizers). /// The name of the regularizer function applied to the output of the layer (its "activation") (see keras.regularizers). /// A tensor of rank 4+ representing activation(conv2d(inputs, kernel) + bias). - public Conv2D Conv2D(int filters, + public ILayer Conv2D(int filters, Shape kernel_size = null, Shape strides = null, string padding = "valid", @@ -228,7 +230,7 @@ public Conv2D Conv2D(int filters, /// The name of the regularizer function applied to the bias vector (see keras.regularizers). /// The name of the regularizer function applied to the output of the layer (its "activation") (see keras.regularizers). /// A tensor of rank 4+ representing activation(conv2d(inputs, kernel) + bias). - public Conv2DTranspose Conv2DTranspose(int filters, + public ILayer Conv2DTranspose(int filters, Shape kernel_size = null, Shape strides = null, string output_padding = "valid", @@ -270,7 +272,7 @@ public Conv2DTranspose Conv2DTranspose(int filters, /// Initializer for the bias vector. /// N-D tensor with shape: (batch_size, ..., input_dim). The most common situation would be a 2D input with shape (batch_size, input_dim). /// N-D tensor with shape: (batch_size, ..., units). For instance, for a 2D input with shape (batch_size, input_dim), the output would have shape (batch_size, units). - public Dense Dense(int units, + public ILayer Dense(int units, Activation activation = null, IInitializer kernel_initializer = null, bool use_bias = true, @@ -294,7 +296,7 @@ public Dense Dense(int units, /// /// Positive integer, dimensionality of the output space. /// N-D tensor with shape: (batch_size, ..., units). For instance, for a 2D input with shape (batch_size, input_dim), the output would have shape (batch_size, units). - public Dense Dense(int units) + public ILayer Dense(int units) => new Dense(new DenseArgs { Units = units, @@ -312,7 +314,7 @@ public Dense Dense(int units) /// Activation function to use. If you don't specify anything, no activation is applied (ie. "linear" activation: a(x) = x). /// N-D tensor with shape: (batch_size, ..., input_dim). The most common situation would be a 2D input with shape (batch_size, input_dim). /// N-D tensor with shape: (batch_size, ..., units). For instance, for a 2D input with shape (batch_size, input_dim), the output would have shape (batch_size, units). - public Dense Dense(int units, + public ILayer Dense(int units, string activation = null, Shape input_shape = null) => new Dense(new DenseArgs @@ -364,7 +366,7 @@ public Tensor dense(Tensor inputs, } - public EinsumDense EinsumDense(string equation, + public ILayer EinsumDense(string equation, Shape output_shape, string bias_axes, Activation activation = null, @@ -402,7 +404,7 @@ public EinsumDense EinsumDense(string equation, /// /// An integer to use as random seed. /// - public Dropout Dropout(float rate, Shape noise_shape = null, int? seed = null) + public ILayer Dropout(float rate, Shape noise_shape = null, int? seed = null) => new Dropout(new DropoutArgs { Rate = rate, @@ -421,7 +423,7 @@ public Dropout Dropout(float rate, Shape noise_shape = null, int? seed = null) /// Initializer for the embeddings matrix (see keras.initializers). /// /// - public Embedding Embedding(int input_dim, + public ILayer Embedding(int input_dim, int output_dim, IInitializer embeddings_initializer = null, bool mask_zero = false, @@ -446,7 +448,7 @@ public Embedding Embedding(int input_dim, /// If you never set it, then it will be "channels_last". /// /// - public Flatten Flatten(string data_format = null) + public ILayer Flatten(string data_format = null) => new Flatten(new FlattenArgs { DataFormat = data_format @@ -482,7 +484,7 @@ public Tensors Input(Shape shape, return input_layer.InboundNodes[0].Outputs; } - public InputLayer InputLayer(Shape input_shape, + public ILayer InputLayer(Shape input_shape, string name = null, bool sparse = false, bool ragged = false) @@ -502,7 +504,7 @@ public InputLayer InputLayer(Shape input_shape, /// /// /// - public AveragePooling2D AveragePooling2D(Shape pool_size = null, + public ILayer AveragePooling2D(Shape pool_size = null, Shape strides = null, string padding = "valid", string data_format = null) @@ -527,7 +529,7 @@ public AveragePooling2D AveragePooling2D(Shape pool_size = null, /// channels_last corresponds to inputs with shape (batch, steps, features) while channels_first corresponds to inputs with shape (batch, features, steps). /// /// - public MaxPooling1D MaxPooling1D(int? pool_size = null, + public ILayer MaxPooling1D(int? pool_size = null, int? strides = null, string padding = "valid", string data_format = null) @@ -564,7 +566,7 @@ public MaxPooling1D MaxPooling1D(int? pool_size = null, /// It defaults to the image_data_format value found in your Keras config file at ~/.keras/keras.json. /// If you never set it, then it will be "channels_last" /// - public MaxPooling2D MaxPooling2D(Shape pool_size = null, + public ILayer MaxPooling2D(Shape pool_size = null, Shape strides = null, string padding = "valid", string data_format = null) @@ -618,7 +620,7 @@ public Tensor max_pooling2d(Tensor inputs, return layer.Apply(inputs); } - public Layer LayerNormalization(Axis? axis, + public ILayer LayerNormalization(Axis? axis, float epsilon = 1e-3f, bool center = true, bool scale = true, @@ -638,45 +640,30 @@ public Layer LayerNormalization(Axis? axis, /// /// Negative slope coefficient. /// - public Layer LeakyReLU(float alpha = 0.3f) + public ILayer LeakyReLU(float alpha = 0.3f) => new LeakyReLu(new LeakyReLuArgs { Alpha = alpha }); - /// - /// Fully-connected RNN where the output is to be fed back to input. - /// - /// Positive integer, dimensionality of the output space. - /// - public Layer SimpleRNN(int units) => SimpleRNN(units, "tanh"); - - /// - /// Fully-connected RNN where the output is to be fed back to input. - /// - /// Positive integer, dimensionality of the output space. - /// Activation function to use. If you pass null, no activation is applied (ie. "linear" activation: a(x) = x). - /// - public Layer SimpleRNN(int units, - Activation activation = null) - => new SimpleRNN(new SimpleRNNArgs - { - Units = units, - Activation = activation - }); - /// /// /// /// Positive integer, dimensionality of the output space. /// The name of the activation function to use. Default: hyperbolic tangent (tanh).. /// - public Layer SimpleRNN(int units, - string activation = "tanh") + public ILayer SimpleRNN(int units, + string activation = "tanh", + string kernel_initializer = "glorot_uniform", + string recurrent_initializer = "orthogonal", + string bias_initializer = "zeros") => new SimpleRNN(new SimpleRNNArgs { Units = units, - Activation = GetActivationByName(activation) + Activation = GetActivationByName(activation), + KernelInitializer = GetInitializerByName(kernel_initializer), + RecurrentInitializer= GetInitializerByName(recurrent_initializer), + BiasInitializer= GetInitializerByName(bias_initializer) }); /// @@ -706,7 +693,7 @@ public Layer SimpleRNN(int units, /// although it tends to be more memory-intensive. Unrolling is only suitable for short sequences. /// /// - public Layer LSTM(int units, + public ILayer LSTM(int units, Activation activation = null, Activation recurrent_activation = null, bool use_bias = true, @@ -749,7 +736,7 @@ public Layer LSTM(int units, /// /// /// - public Rescaling Rescaling(float scale, + public ILayer Rescaling(float scale, float offset = 0, Shape input_shape = null) => new Rescaling(new RescalingArgs @@ -763,21 +750,21 @@ public Rescaling Rescaling(float scale, /// /// /// - public Add Add() + public ILayer Add() => new Add(new MergeArgs { }); /// /// /// /// - public Subtract Subtract() + public ILayer Subtract() => new Subtract(new MergeArgs { }); /// /// Global max pooling operation for spatial data. /// /// - public GlobalAveragePooling2D GlobalAveragePooling2D() + public ILayer GlobalAveragePooling2D() => new GlobalAveragePooling2D(new Pooling2DArgs { }); /// @@ -787,7 +774,7 @@ public GlobalAveragePooling2D GlobalAveragePooling2D() /// channels_last corresponds to inputs with shape (batch, steps, features) while channels_first corresponds to inputs with shape (batch, features, steps). /// /// - public GlobalAveragePooling1D GlobalAveragePooling1D(string data_format = "channels_last") + public ILayer GlobalAveragePooling1D(string data_format = "channels_last") => new GlobalAveragePooling1D(new Pooling1DArgs { DataFormat = data_format }); /// @@ -796,7 +783,7 @@ public GlobalAveragePooling1D GlobalAveragePooling1D(string data_format = "chann /// A string, one of channels_last (default) or channels_first. The ordering of the dimensions in the inputs. /// channels_last corresponds to inputs with shape (batch, height, width, channels) while channels_first corresponds to inputs with shape (batch, channels, height, width). /// - public GlobalAveragePooling2D GlobalAveragePooling2D(string data_format = "channels_last") + public ILayer GlobalAveragePooling2D(string data_format = "channels_last") => new GlobalAveragePooling2D(new Pooling2DArgs { DataFormat = data_format }); /// @@ -807,7 +794,7 @@ public GlobalAveragePooling2D GlobalAveragePooling2D(string data_format = "chann /// channels_last corresponds to inputs with shape (batch, steps, features) while channels_first corresponds to inputs with shape (batch, features, steps). /// /// - public GlobalMaxPooling1D GlobalMaxPooling1D(string data_format = "channels_last") + public ILayer GlobalMaxPooling1D(string data_format = "channels_last") => new GlobalMaxPooling1D(new Pooling1DArgs { DataFormat = data_format }); /// @@ -816,7 +803,7 @@ public GlobalMaxPooling1D GlobalMaxPooling1D(string data_format = "channels_last /// A string, one of channels_last (default) or channels_first. The ordering of the dimensions in the inputs. /// channels_last corresponds to inputs with shape (batch, height, width, channels) while channels_first corresponds to inputs with shape (batch, channels, height, width). /// - public GlobalMaxPooling2D GlobalMaxPooling2D(string data_format = "channels_last") + public ILayer GlobalMaxPooling2D(string data_format = "channels_last") => new GlobalMaxPooling2D(new Pooling2DArgs { DataFormat = data_format }); @@ -848,6 +835,7 @@ IInitializer GetInitializerByName(string name) "glorot_uniform" => tf.glorot_uniform_initializer, "zeros" => tf.zeros_initializer, "ones" => tf.ones_initializer, + "orthogonal" => tf.orthogonal_initializer, _ => tf.glorot_uniform_initializer }; } diff --git a/src/TensorFlowNET.Keras/Layers/LSTM.cs b/src/TensorFlowNET.Keras/Layers/Lstm/LSTM.cs similarity index 87% rename from src/TensorFlowNET.Keras/Layers/LSTM.cs rename to src/TensorFlowNET.Keras/Layers/Lstm/LSTM.cs index 73a2df121..b7d973847 100644 --- a/src/TensorFlowNET.Keras/Layers/LSTM.cs +++ b/src/TensorFlowNET.Keras/Layers/Lstm/LSTM.cs @@ -1,8 +1,9 @@ using System.Linq; -using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.ArgsDefinition.Lstm; using Tensorflow.Keras.Engine; +using Tensorflow.Keras.Layers.Rnn; -namespace Tensorflow.Keras.Layers +namespace Tensorflow.Keras.Layers.Lstm { /// /// Long Short-Term Memory layer - Hochreiter 1997. diff --git a/src/TensorFlowNET.Keras/Layers/LSTMCell.cs b/src/TensorFlowNET.Keras/Layers/Lstm/LSTMCell.cs similarity index 72% rename from src/TensorFlowNET.Keras/Layers/LSTMCell.cs rename to src/TensorFlowNET.Keras/Layers/Lstm/LSTMCell.cs index dda279a79..3cd35a091 100644 --- a/src/TensorFlowNET.Keras/Layers/LSTMCell.cs +++ b/src/TensorFlowNET.Keras/Layers/Lstm/LSTMCell.cs @@ -1,7 +1,7 @@ -using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.ArgsDefinition.Lstm; using Tensorflow.Keras.Engine; -namespace Tensorflow.Keras.Layers +namespace Tensorflow.Keras.Layers.Lstm { public class LSTMCell : Layer { diff --git a/src/TensorFlowNET.Keras/Layers/RNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs similarity index 95% rename from src/TensorFlowNET.Keras/Layers/RNN.cs rename to src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs index 293c27fb6..c2b86ae4f 100644 --- a/src/TensorFlowNET.Keras/Layers/RNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs @@ -1,10 +1,12 @@ using System; using System.Collections.Generic; using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; +using Tensorflow.Keras.Layers.Lstm; // from tensorflow.python.distribute import distribution_strategy_context as ds_context; -namespace Tensorflow.Keras.Layers +namespace Tensorflow.Keras.Layers.Rnn { public class RNN : Layer { @@ -14,6 +16,8 @@ public class RNN : Layer private object _states = null; private object constants_spec = null; private int _num_constants = 0; + protected IVariableV1 kernel; + protected IVariableV1 bias; public RNN(RNNArgs args) : base(PreConstruct(args)) { diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs new file mode 100644 index 000000000..58b700fe6 --- /dev/null +++ b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs @@ -0,0 +1,31 @@ +using System.Data; +using Tensorflow.Keras.ArgsDefinition.Rnn; +using Tensorflow.Operations.Activation; +using static HDF.PInvoke.H5Z; +using static Tensorflow.ApiDef.Types; + +namespace Tensorflow.Keras.Layers.Rnn +{ + public class SimpleRNN : RNN + { + SimpleRNNArgs args; + SimpleRNNCell cell; + public SimpleRNN(SimpleRNNArgs args) : base(args) + { + this.args = args; + } + + protected override void build(Tensors inputs) + { + var input_shape = inputs.shape; + var input_dim = input_shape[-1]; + + kernel = add_weight("kernel", (input_shape[-1], args.Units), + initializer: args.KernelInitializer + //regularizer = self.kernel_regularizer, + //constraint = self.kernel_constraint, + //caching_device = default_caching_device, + ); + } + } +} \ No newline at end of file diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs new file mode 100644 index 000000000..de50c3618 --- /dev/null +++ b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Keras.ArgsDefinition.Rnn; +using Tensorflow.Keras.Engine; + +namespace Tensorflow.Keras.Layers.Rnn +{ + public class SimpleRNNCell : Layer + { + public SimpleRNNCell(SimpleRNNArgs args) : base(args) + { + + } + + protected override void build(Tensors inputs) + { + + } + } +} diff --git a/src/TensorFlowNET.Keras/Layers/StackedRNNCells.cs b/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs similarity index 98% rename from src/TensorFlowNET.Keras/Layers/StackedRNNCells.cs rename to src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs index 2da206ca8..eead274a1 100644 --- a/src/TensorFlowNET.Keras/Layers/StackedRNNCells.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs @@ -2,9 +2,10 @@ using System.Collections.Generic; using System.ComponentModel; using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; -namespace Tensorflow.Keras.Layers +namespace Tensorflow.Keras.Layers.Rnn { public class StackedRNNCells : Layer, RNNArgs.IRnnArgCell { diff --git a/src/TensorFlowNET.Keras/Layers/SimpleRNN.cs b/src/TensorFlowNET.Keras/Layers/SimpleRNN.cs deleted file mode 100644 index c1fc4afd6..000000000 --- a/src/TensorFlowNET.Keras/Layers/SimpleRNN.cs +++ /dev/null @@ -1,14 +0,0 @@ -using Tensorflow.Keras.ArgsDefinition; - -namespace Tensorflow.Keras.Layers -{ - public class SimpleRNN : RNN - { - - public SimpleRNN(RNNArgs args) : base(args) - { - - } - - } -} \ No newline at end of file diff --git a/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.Resizing.cs b/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.Resizing.cs index 5e93f5836..0be7f1e6c 100644 --- a/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.Resizing.cs +++ b/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.Resizing.cs @@ -15,7 +15,7 @@ public partial class Preprocessing /// /// /// - public Resizing Resizing(int height, int width, string interpolation = "bilinear") + public ILayer Resizing(int height, int width, string interpolation = "bilinear") => new Resizing(new ResizingArgs { Height = height, diff --git a/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.cs b/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.cs index 994a36d6c..94fc4a207 100644 --- a/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.cs +++ b/src/TensorFlowNET.Keras/Preprocessings/Preprocessing.cs @@ -5,7 +5,7 @@ namespace Tensorflow.Keras { - public partial class Preprocessing + public partial class Preprocessing : IPreprocessing { public Sequence sequence => new Sequence(); public DatasetUtils dataset_utils => new DatasetUtils(); @@ -14,7 +14,7 @@ public partial class Preprocessing private static TextApi _text = new TextApi(); - public TextVectorization TextVectorization(Func standardize = null, + public ILayer TextVectorization(Func standardize = null, string split = "whitespace", int max_tokens = -1, string output_mode = "int", diff --git a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj index 3d4484543..0c3eff9fb 100644 --- a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj +++ b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj @@ -3,11 +3,11 @@ netstandard2.0 Tensorflow.Keras - 9.0 + 10.0 enable Tensorflow.Keras AnyCPU;x64 - 0.7.0 + 0.10.0 Haiping Chen Keras for .NET Apache 2.0, Haiping Chen 2021 @@ -37,9 +37,10 @@ Keras is an API designed for human beings, not machines. Keras follows best prac Git true Open.snk - 0.7.0.0 - 0.7.0.0 + 0.10.0.0 + 0.10.0.0 LICENSE + Debug;Release;GPU @@ -47,6 +48,11 @@ Keras is an API designed for human beings, not machines. Keras follows best prac false + + DEBUG;TRACE + false + + false @@ -55,6 +61,10 @@ Keras is an API designed for human beings, not machines. Keras follows best prac Tensorflow.Keras.xml + + Tensorflow.Keras.xml + + diff --git a/src/TensorFlowNET.Keras/tf.layers.cs b/src/TensorFlowNET.Keras/tf.layers.cs index 3f5ed01ca..da7c23471 100644 --- a/src/TensorFlowNET.Keras/tf.layers.cs +++ b/src/TensorFlowNET.Keras/tf.layers.cs @@ -134,7 +134,7 @@ public Tensors batch_normalization(Tensor inputs, /// /// /// - public Tensor max_pooling2d(Tensor inputs, + public Tensor MaxPooling2D(Tensor inputs, int[] pool_size, int[] strides, string padding = "valid", diff --git a/src/python/.vscode/launch.json b/src/python/.vscode/launch.json new file mode 100644 index 000000000..2b2502c69 --- /dev/null +++ b/src/python/.vscode/launch.json @@ -0,0 +1,16 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Current File", + "type": "python", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "justMyCode": false + } + ] +} \ No newline at end of file diff --git a/src/python/simple_rnn.py b/src/python/simple_rnn.py new file mode 100644 index 000000000..97f9f3f31 --- /dev/null +++ b/src/python/simple_rnn.py @@ -0,0 +1,15 @@ +import numpy as np +import tensorflow as tf + +# tf.experimental.numpy +inputs = np.random.random([32, 10, 8]).astype(np.float32) +simple_rnn = tf.keras.layers.SimpleRNN(4) + +output = simple_rnn(inputs) # The output has shape `[32, 4]`. + +simple_rnn = tf.keras.layers.SimpleRNN( + 4, return_sequences=True, return_state=True) + +# whole_sequence_output has shape `[32, 10, 4]`. +# final_state has shape `[32, 4]`. +whole_sequence_output, final_state = simple_rnn(inputs) \ No newline at end of file diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/AttentionTest.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/AttentionTest.cs index 0c02b5db1..02298ce81 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/AttentionTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/AttentionTest.cs @@ -83,7 +83,7 @@ public void test_calculate_scores_multi_dim() { 2.5f, 2.6f, 2.7f, 2.8f }, { 3.5f, 3.6f, 3.7f, 3.8f } } }, dtype: np.float32); - var attention_layer = keras.layers.Attention(); + var attention_layer = (Attention)keras.layers.Attention(); //attention_layer.build(((1, 2, 4), (1, 3, 4))); var actual = attention_layer._calculate_scores(query: q, key: k); // Expected tensor of shape [1, 2, 3]. @@ -116,7 +116,7 @@ public void test_calculate_scores_multi_dim_concat() { 2.5f, 2.6f, 2.7f, 2.8f }, { 3.5f, 3.6f, 3.7f, 3.8f } } }, dtype: np.float32); - var attention_layer = keras.layers.Attention(score_mode: "concat"); + var attention_layer = (Attention)keras.layers.Attention(score_mode: "concat"); //attention_layer.concat_score_weight = 1; attention_layer.concat_score_weight = base_layer_utils.make_variable(new VariableArgs() { Name = "concat_score_weight", diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs index 53a13394f..f4fdf94a5 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs @@ -148,10 +148,9 @@ public void EinsumDense() } [TestMethod] - [Ignore] public void SimpleRNN() { - var inputs = np.random.rand(32, 10, 8).astype(np.float32); + var inputs = np.random.random((32, 10, 8)).astype(np.float32); var simple_rnn = keras.layers.SimpleRNN(4); var output = simple_rnn.Apply(inputs); Assert.AreEqual((32, 4), output.shape); diff --git a/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj b/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj index 6d0b1ca35..fc693b1ef 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj +++ b/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj @@ -4,7 +4,7 @@ net6.0 false - + 11.0 AnyCPU;x64 diff --git a/test/TensorFlowNET.UnitTest/Tensorflow.Binding.UnitTest.csproj b/test/TensorFlowNET.UnitTest/Tensorflow.Binding.UnitTest.csproj index ffb583c94..36ff4a3dd 100644 --- a/test/TensorFlowNET.UnitTest/Tensorflow.Binding.UnitTest.csproj +++ b/test/TensorFlowNET.UnitTest/Tensorflow.Binding.UnitTest.csproj @@ -11,7 +11,7 @@ Open.snk - 9.0 + 11.0 AnyCPU;x64 From 9b11d459069bd2cb7fcadcef402e2dabd4d1090d Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sat, 19 Nov 2022 10:04:02 -0600 Subject: [PATCH 02/52] Fix NeuralNetXorKeras accuracy. #952 --- src/TensorFlowNET.Keras/Engine/Functional.cs | 2 +- .../Engine/Layer.AddWeights.cs | 4 +- src/TensorFlowNET.Keras/Engine/Layer.cs | 20 ++++---- src/TensorFlowNET.Keras/Engine/Model.Fit.cs | 47 ++++++++++++++++--- src/TensorFlowNET.Keras/Engine/Model.cs | 15 ++++++ src/python/.vscode/launch.json | 2 +- src/python/xor_keras.py | 23 +++++++++ 7 files changed, 92 insertions(+), 21 deletions(-) create mode 100644 src/python/xor_keras.py diff --git a/src/TensorFlowNET.Keras/Engine/Functional.cs b/src/TensorFlowNET.Keras/Engine/Functional.cs index 01d84794f..def842c32 100644 --- a/src/TensorFlowNET.Keras/Engine/Functional.cs +++ b/src/TensorFlowNET.Keras/Engine/Functional.cs @@ -71,7 +71,7 @@ protected void _init_graph_network(Tensors inputs, Tensors outputs) NodesByDepth = nodes_by_depth; if (_layers.Count == 0) _layers = layers; - + _self_tracked_trackables = layers; // Build self.input_names and self.output_names. _set_output_names(); diff --git a/src/TensorFlowNET.Keras/Engine/Layer.AddWeights.cs b/src/TensorFlowNET.Keras/Engine/Layer.AddWeights.cs index feb5e8e40..703e7f23b 100644 --- a/src/TensorFlowNET.Keras/Engine/Layer.AddWeights.cs +++ b/src/TensorFlowNET.Keras/Engine/Layer.AddWeights.cs @@ -53,9 +53,9 @@ protected virtual IVariableV1 add_weight(string name, //backend.track_variable(variable); if (trainable == true) - trainable_weights.Add(variable); + _trainable_weights.Add(variable); else - non_trainable_weights.Add(variable); + _non_trainable_weights.Add(variable); return variable; } diff --git a/src/TensorFlowNET.Keras/Engine/Layer.cs b/src/TensorFlowNET.Keras/Engine/Layer.cs index 03308ede4..38c756065 100644 --- a/src/TensorFlowNET.Keras/Engine/Layer.cs +++ b/src/TensorFlowNET.Keras/Engine/Layer.cs @@ -61,12 +61,12 @@ public abstract partial class Layer : AutoTrackable, ILayer protected InputSpec inputSpec; bool dynamic = true; public bool SupportsMasking { get; set; } - protected List trainable_weights; + protected List _trainable_weights; - public virtual List trainable_variables => trainable_weights; + public virtual List trainable_variables => _trainable_weights; - protected List non_trainable_weights; - public List non_trainable_variables => non_trainable_weights; + protected List _non_trainable_weights; + public List non_trainable_variables => _non_trainable_weights; protected int id; public int Id => id; @@ -104,8 +104,8 @@ public Layer(LayerArgs args) id = ops.uid_layer(); _init_set_name(args.Name); - trainable_weights = new List(); - non_trainable_weights = new List(); + _trainable_weights = new List(); + _non_trainable_weights = new List(); computePreviousMask = false; updates = new List(); _self_tracked_trackables = new List(); @@ -254,7 +254,7 @@ List ILayer.trainable_weights { get { - return trainable_weights; + return _trainable_weights; } } @@ -262,7 +262,7 @@ List ILayer.non_trainable_weights { get { - return non_trainable_weights; + return _non_trainable_weights; } } @@ -271,8 +271,8 @@ public List weights get { var weights = new List(); - weights.AddRange(trainable_weights); - weights.AddRange(non_trainable_weights); + weights.AddRange(_trainable_weights); + weights.AddRange(_non_trainable_weights); return weights; } set diff --git a/src/TensorFlowNET.Keras/Engine/Model.Fit.cs b/src/TensorFlowNET.Keras/Engine/Model.Fit.cs index ab4ba0dec..db86db633 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Fit.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Fit.cs @@ -4,6 +4,7 @@ using System.Linq; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine.DataAdapters; +using System.Diagnostics; namespace Tensorflow.Keras.Engine { @@ -87,25 +88,57 @@ void FitInternal(int epochs, int verbose) { stop_training = false; _train_counter.assign(0); + Stopwatch sw = new Stopwatch(); foreach (var (epoch, iterator) in data_handler.enumerate_epochs()) { reset_metrics(); - // callbacks.on_epoch_begin(epoch) + on_epoch_begin(epoch, epochs); // data_handler.catch_stop_iteration(); foreach (var step in data_handler.steps()) { - // callbacks.on_train_batch_begin(step) + sw.Start(); var results = train_step_function(iterator); - if (verbose == 1) + sw.Stop(); + on_train_batch_begin(verbose, step, sw.ElapsedMilliseconds, results); + + // recycle memory more frequency + if (sw.ElapsedMilliseconds > 100) { - var result_pairs = string.Join(", ", results.Select(x => $"{x.Item1}: {(float)x.Item2:F6}")); - Binding.tf_output_redirect.WriteLine($"Epoch: {epoch + 1:D3}/{epochs:D3}, Step: {step + 1:D4}/{data_handler.Inferredsteps:D4}, {result_pairs}"); + GC.Collect(); } - - GC.Collect(); + sw.Reset(); } + Console.WriteLine(); + + GC.Collect(); GC.WaitForPendingFinalizers(); } } + + void on_epoch_begin(int epoch, int epochs) + { + Binding.tf_output_redirect.WriteLine($"Epoch: {epoch + 1:D3}/{epochs:D3}"); + } + + void on_train_batch_begin(int verbose, long step, long elapse, IEnumerable<(string, Tensor)> results) + { + if (verbose == 1) + { + var result_pairs = string.Join(", ", results.Select(x => $"{x.Item1}: {(float)x.Item2:F6}")); + + var progress = ""; + for (int i = 0; i < step + 1; i++) + for (int j = 0; j < 30 / data_handler.Inferredsteps; j++) + progress += "="; + progress += ">"; + + var remaining = ""; + for (int i = 1; i < 30 - progress.Length; i++) + remaining += "."; + + Binding.tf_output_redirect.Write($"{step + 1:D4}/{data_handler.Inferredsteps:D4} [{progress}{remaining}] - {elapse}ms/step {result_pairs}"); + Console.CursorLeft = 0; + } + } } } diff --git a/src/TensorFlowNET.Keras/Engine/Model.cs b/src/TensorFlowNET.Keras/Engine/Model.cs index 9e38d59ac..4ae94b3dc 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.cs @@ -75,11 +75,26 @@ public override List trainable_variables get { var variables = new List(); + + if (!Trainable) + { + return variables; + } + + foreach (var trackable_obj in _self_tracked_trackables) + { + if (trackable_obj.Trainable) + variables.AddRange(trackable_obj.trainable_variables); + } + foreach (var layer in _layers) { if (layer.Trainable) variables.AddRange(layer.trainable_variables); } + + // variables.AddRange(_trainable_weights); + return variables; } } diff --git a/src/python/.vscode/launch.json b/src/python/.vscode/launch.json index 2b2502c69..4d4e27495 100644 --- a/src/python/.vscode/launch.json +++ b/src/python/.vscode/launch.json @@ -8,7 +8,7 @@ "name": "Python: Current File", "type": "python", "request": "launch", - "program": "${file}", + "program": "${workspaceFolder}/xor_keras.py", "console": "integratedTerminal", "justMyCode": false } diff --git a/src/python/xor_keras.py b/src/python/xor_keras.py new file mode 100644 index 000000000..ffd88b612 --- /dev/null +++ b/src/python/xor_keras.py @@ -0,0 +1,23 @@ +import os +import numpy as np +import tensorflow as tf + +os.environ["CUDA_VISIBLE_DEVICES"] = "-1" +print(tf.__version__) +# tf.compat.v1.enable_eager_execution() +# tf.debugging.set_log_device_placement(True); +tf.config.run_functions_eagerly(True) + +x = np.array([[ 0, 0 ], [ 0, 1 ], [ 1, 0 ], [ 1, 1 ]]) +y = np.array([[ 0 ], [ 1 ], [ 1 ], [ 0 ] ]) + +model = tf.keras.Sequential() +model.add(tf.keras.Input(2)) +model.add(tf.keras.layers.Dense(32, "relu")) +model.add(tf.keras.layers.Dense(1, "sigmoid")) +model.compile(optimizer = tf.keras.optimizers.Adam(), + loss = tf.keras.losses.MeanSquaredError(), + metrics = ["accuracy"]) +model.fit(x, y, 1, 100) +result = model.evaluate(x, y) +print(model.predict(x, 4)) \ No newline at end of file From def066498d28d1f64626bfff212fa16207db3704 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 24 Nov 2022 18:31:49 -0600 Subject: [PATCH 03/52] Fix NeuralNetXor example. --- .../Tensorflow.Binding.csproj | 4 +- .../Optimizers/OptimizerV2.cs | 2 +- src/TensorFlowNET.Keras/Saving/hdf5_format.cs | 41 ++++++++----------- .../Tensorflow.Keras.csproj | 6 +-- 4 files changed, 23 insertions(+), 30 deletions(-) diff --git a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj index 364498268..0ebe61d0d 100644 --- a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj +++ b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj @@ -107,8 +107,8 @@ https://tensorflownet.readthedocs.io - + - + diff --git a/src/TensorFlowNET.Keras/Optimizers/OptimizerV2.cs b/src/TensorFlowNET.Keras/Optimizers/OptimizerV2.cs index 73e35d028..a52c5ada5 100644 --- a/src/TensorFlowNET.Keras/Optimizers/OptimizerV2.cs +++ b/src/TensorFlowNET.Keras/Optimizers/OptimizerV2.cs @@ -48,7 +48,7 @@ public OptimizerV2(OptimizerV2Args args) : base() public void apply_gradients((Tensor, ResourceVariable) grads_and_vars, string name = null, bool experimental_aggregate_gradients = true) - => apply_gradients(grads_and_vars, + => apply_gradients(new[] { grads_and_vars }, name: name, experimental_aggregate_gradients: experimental_aggregate_gradients); diff --git a/src/TensorFlowNET.Keras/Saving/hdf5_format.cs b/src/TensorFlowNET.Keras/Saving/hdf5_format.cs index 0c3404772..a3705dfba 100644 --- a/src/TensorFlowNET.Keras/Saving/hdf5_format.cs +++ b/src/TensorFlowNET.Keras/Saving/hdf5_format.cs @@ -84,23 +84,18 @@ public static void load_weights_from_hdf5_group(long f, List layers) { string original_keras_version = "2.5.0"; string original_backend = null; - if (Hdf5.AttributeExists(f, "keras_version")) - { - var (success, attr) = Hdf5.ReadStringAttributes(f, "keras_version", ""); - if (success) - original_keras_version = attr.First(); - // keras version should be 2.5.0+ - var ver_major = int.Parse(original_keras_version.Split('.')[0]); - var ver_minor = int.Parse(original_keras_version.Split('.')[1]); - if (ver_major < 2 || (ver_major == 2 && ver_minor < 5)) - throw new ValueError("keras version should be 2.5.0 or later."); - } - if (Hdf5.AttributeExists(f, "backend")) - { - var (success, attr) = Hdf5.ReadStringAttributes(f, "backend", ""); - if (success) - original_backend = attr.First(); - } + var (success, attr) = Hdf5.ReadStringAttributes(f, "keras_version", "", true); + if (success) + original_keras_version = attr.First(); + // keras version should be 2.5.0+ + var ver_major = int.Parse(original_keras_version.Split('.')[0]); + var ver_minor = int.Parse(original_keras_version.Split('.')[1]); + if (ver_major < 2 || (ver_major == 2 && ver_minor < 5)) + throw new ValueError("keras version should be 2.5.0 or later."); + + (success, attr) = Hdf5.ReadStringAttributes(f, "backend", "", true); + if (success) + original_backend = attr.First(); var filtered_layers = new List(); foreach (var layer in layers) @@ -137,7 +132,7 @@ public static void load_weights_from_hdf5_group(long f, List layers) var weight_names = load_attributes_from_hdf5_group(g, "weight_names"); foreach (var i_ in weight_names) { - (bool success, Array result) = Hdf5.ReadDataset(g, i_); + (success, Array result) = Hdf5.ReadDataset(g, i_); if (success) weight_values.Add(np.array(result)); } @@ -329,12 +324,10 @@ private static List> Split(Array list, int chunkSize) public static string[] load_attributes_from_hdf5_group(long group, string name) { - if (Hdf5.AttributeExists(group, name)) - { - var (success, attr) = Hdf5.ReadStringAttributes(group, name, ""); - if (success) - return attr.ToArray(); - } + var (success, attr) = Hdf5.ReadStringAttributes(group, name, "", true); + if (success) + return attr.ToArray(); + return null; } diff --git a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj index 0c3eff9fb..23c996208 100644 --- a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj +++ b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj @@ -70,10 +70,10 @@ Keras is an API designed for human beings, not machines. Keras follows best prac - - + + - + From 10f66f0bafa235338c35cbf9faa5f14ba675b4c0 Mon Sep 17 00:00:00 2001 From: AsakusaRinne Date: Sat, 26 Nov 2022 04:55:03 +0800 Subject: [PATCH 04/52] Fix IO Exception from keras.fit. --- src/TensorFlowNET.Keras/Engine/Model.Fit.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/TensorFlowNET.Keras/Engine/Model.Fit.cs b/src/TensorFlowNET.Keras/Engine/Model.Fit.cs index db86db633..e0b4af78c 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Fit.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Fit.cs @@ -137,7 +137,10 @@ void on_train_batch_begin(int verbose, long step, long elapse, IEnumerable<(stri remaining += "."; Binding.tf_output_redirect.Write($"{step + 1:D4}/{data_handler.Inferredsteps:D4} [{progress}{remaining}] - {elapse}ms/step {result_pairs}"); - Console.CursorLeft = 0; + if (!Console.IsOutputRedirected) + { + Console.CursorLeft = 0; + } } } } From 0a08386ca95aaa5bc50cf581f97e5c611cdd5fcf Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sat, 26 Nov 2022 16:01:14 -0600 Subject: [PATCH 05/52] Fix batch_size for Keras Input. --- src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs | 1 + src/TensorFlowNET.Keras/Layers/LayersApi.cs | 2 ++ src/python/xor_keras.py | 1 + 3 files changed, 4 insertions(+) diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs index 5945bb551..3f4d1ed8e 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs @@ -108,6 +108,7 @@ public ILayer EinsumDense(string equation, public ILayer GlobalMaxPooling2D(string data_format = "channels_last"); public Tensors Input(Shape shape, + int batch_size = -1, string name = null, bool sparse = false, bool ragged = false); diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index 8498f5ac6..50c66be70 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -469,6 +469,7 @@ public ILayer Flatten(string data_format = null) /// /// A tensor. public Tensors Input(Shape shape, + int batch_size = -1, string name = null, bool sparse = false, bool ragged = false) @@ -476,6 +477,7 @@ public Tensors Input(Shape shape, var input_layer = new InputLayer(new InputLayerArgs { InputShape = shape, + BatchSize= batch_size, Name = name, Sparse = sparse, Ragged = ragged diff --git a/src/python/xor_keras.py b/src/python/xor_keras.py index ffd88b612..e73886050 100644 --- a/src/python/xor_keras.py +++ b/src/python/xor_keras.py @@ -4,6 +4,7 @@ os.environ["CUDA_VISIBLE_DEVICES"] = "-1" print(tf.__version__) +# https://playground.tensorflow.org/ # tf.compat.v1.enable_eager_execution() # tf.debugging.set_log_device_placement(True); tf.config.run_functions_eagerly(True) From 2adfcd2cf92a3049da69dfed494603c804df6f18 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sat, 3 Dec 2022 12:44:00 -0600 Subject: [PATCH 06/52] Fix Sequential model.summary missing layers. #960 --- src/TensorFlowNET.Keras/Engine/Layer.FlattenLayers.cs | 2 +- src/TensorFlowNET.Keras/Engine/Layer.Layers.cs | 2 +- src/TensorFlowNET.Keras/Engine/Model.cs | 4 ++++ src/TensorFlowNET.Keras/Engine/Sequential.cs | 3 +++ 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/TensorFlowNET.Keras/Engine/Layer.FlattenLayers.cs b/src/TensorFlowNET.Keras/Engine/Layer.FlattenLayers.cs index e088fdaf4..dd037e243 100644 --- a/src/TensorFlowNET.Keras/Engine/Layer.FlattenLayers.cs +++ b/src/TensorFlowNET.Keras/Engine/Layer.FlattenLayers.cs @@ -10,7 +10,7 @@ public IEnumerable _flatten_layers(bool recursive = true, bool include_s yield return this; var seen_object_ids = new List(); - var deque = new Queue(_layers); + var deque = new Queue(_self_tracked_trackables); while (!deque.empty()) { var layer_or_container = deque.Dequeue(); diff --git a/src/TensorFlowNET.Keras/Engine/Layer.Layers.cs b/src/TensorFlowNET.Keras/Engine/Layer.Layers.cs index 325358386..f38750c25 100644 --- a/src/TensorFlowNET.Keras/Engine/Layer.Layers.cs +++ b/src/TensorFlowNET.Keras/Engine/Layer.Layers.cs @@ -6,7 +6,7 @@ namespace Tensorflow.Keras.Engine public partial class Layer { protected List _layers = new List(); - public List Layers => _layers; + public virtual List Layers => _layers; protected void StackLayers(params ILayer[] layers) { diff --git a/src/TensorFlowNET.Keras/Engine/Model.cs b/src/TensorFlowNET.Keras/Engine/Model.cs index 4ae94b3dc..baf68229a 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.Linq; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine.DataAdapters; using Tensorflow.Keras.Losses; @@ -70,6 +71,9 @@ void _init_batch_counters() aggregation: VariableAggregation.OnlyFirstReplica); } + public override List Layers + => _flatten_layers(recursive: false, include_self: false).ToList(); + public override List trainable_variables { get diff --git a/src/TensorFlowNET.Keras/Engine/Sequential.cs b/src/TensorFlowNET.Keras/Engine/Sequential.cs index 7d8c77fea..47e6c3f77 100644 --- a/src/TensorFlowNET.Keras/Engine/Sequential.cs +++ b/src/TensorFlowNET.Keras/Engine/Sequential.cs @@ -202,5 +202,8 @@ void track_nodes_created_by_last_call(ILayer layer, List created_nodes) created_nodes.add(prev_layer.OutboundNodes.Last()); } } + + public override List Layers + => base.Layers.Where(x => x is not InputLayer).ToList(); } } From 49876bb9520b8999e14573e18fb3c30c1cc7e7df Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sun, 4 Dec 2022 23:04:57 -0600 Subject: [PATCH 07/52] Change model.build parameter to input_shape. --- src/TensorFlowNET.Keras/Engine/Layer.cs | 4 +- .../Layers/Activation/ELU.cs | 58 ++++++------ .../Layers/Activation/Exponential.cs | 35 ++++---- .../Layers/Activation/SELU.cs | 2 +- .../Layers/Attention/Attention.cs | 7 +- .../Layers/Convolution/Conv2DTranspose.cs | 9 +- .../Layers/Convolution/Convolutional.cs | 3 +- src/TensorFlowNET.Keras/Layers/Core/Dense.cs | 3 +- .../Layers/Core/EinsumDense.cs | 5 +- .../Layers/Core/Embedding.cs | 2 +- .../Layers/Cropping/Cropping1D.cs | 90 +++++++++++-------- .../Layers/Cropping/Cropping2D.cs | 2 +- .../Layers/Cropping/Cropping3D.cs | 2 +- .../Layers/Merging/Concatenate.cs | 2 +- .../Layers/Merging/Merge.cs | 2 +- .../Normalization/BatchNormalization.cs | 3 +- .../Normalization/LayerNormalization.cs | 3 +- .../Layers/Preprocessing/TextVectorization.cs | 6 +- .../Layers/Reshaping/Permute.cs | 59 ++++++------ .../Layers/Rnn/SimpleRNN.cs | 3 +- .../Layers/Rnn/SimpleRNNCell.cs | 2 +- 21 files changed, 163 insertions(+), 139 deletions(-) diff --git a/src/TensorFlowNET.Keras/Engine/Layer.cs b/src/TensorFlowNET.Keras/Engine/Layer.cs index 38c756065..d417fa44b 100644 --- a/src/TensorFlowNET.Keras/Engine/Layer.cs +++ b/src/TensorFlowNET.Keras/Engine/Layer.cs @@ -191,7 +191,7 @@ protected void MaybeBuild(Tensors inputs) tf.Context.eager_mode(isFunc: tf.Context.is_build_function()); } - build(inputs); + build(inputs.shape); if (need_restore_mode) tf.Context.restore_mode(); @@ -199,7 +199,7 @@ protected void MaybeBuild(Tensors inputs) built = true; } - protected virtual void build(Tensors inputs) + public virtual void build(Shape input_shape) { built = true; } diff --git a/src/TensorFlowNET.Keras/Layers/Activation/ELU.cs b/src/TensorFlowNET.Keras/Layers/Activation/ELU.cs index 3efda3649..6e790a26f 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/ELU.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/ELU.cs @@ -6,30 +6,38 @@ using static Tensorflow.Binding; namespace Tensorflow.Keras.Layers { - /// - /// ELU Layer: - /// x = 0 when x > 0, x = alpha( e^x-1 ) elsewhere - /// - public class ELU : Layer { - ELUArgs args; - float alpha => args.Alpha; - public ELU ( ELUArgs args ) : base(args) { - this.args = args; - } - protected override void build ( Tensors inputs ) { - if ( alpha < 0f ) { - throw new ValueError("Alpha must be a number greater than 0."); - } - built = true; - } - protected override Tensors Call ( Tensors inputs, Tensor state = null, bool? training = null ) { - Tensor output = inputs; - output = tf.where(output > 0f, output, - tf.multiply(alpha, tf.sub(tf.exp(output), 1f))); - return output; - } - public override Shape ComputeOutputShape ( Shape input_shape ) { - return input_shape; + /// + /// ELU Layer: + /// x = 0 when x > 0, x = alpha( e^x-1 ) elsewhere + /// + public class ELU : Layer + { + ELUArgs args; + float alpha => args.Alpha; + public ELU(ELUArgs args) : base(args) + { + this.args = args; + } + + public override void build(Shape input_shape) + { + if (alpha < 0f) + { + throw new ValueError("Alpha must be a number greater than 0."); } - } + built = true; + } + + protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + { + Tensor output = inputs; + output = tf.where(output > 0f, output, + tf.multiply(alpha, tf.sub(tf.exp(output), 1f))); + return output; + } + public override Shape ComputeOutputShape(Shape input_shape) + { + return input_shape; + } + } } diff --git a/src/TensorFlowNET.Keras/Layers/Activation/Exponential.cs b/src/TensorFlowNET.Keras/Layers/Activation/Exponential.cs index aecb3da24..aba175de9 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/Exponential.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/Exponential.cs @@ -6,19 +6,24 @@ using static Tensorflow.Binding; namespace Tensorflow.Keras.Layers { - public class Exponential : Layer { - public Exponential ( LayerArgs args ) : base(args) { - // Exponential has no args - } - protected override void build ( Tensors inputs ) { - built = true; - } - protected override Tensors Call ( Tensors inputs, Tensor state = null, bool? training = null ) { - Tensor output = inputs; - return tf.exp(output); - } - public override Shape ComputeOutputShape ( Shape input_shape ) { - return input_shape; - } - } + public class Exponential : Layer + { + public Exponential(LayerArgs args) : base(args) + { + // Exponential has no args + } + public override void build(Shape input_shape) + { + built = true; + } + protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + { + Tensor output = inputs; + return tf.exp(output); + } + public override Shape ComputeOutputShape(Shape input_shape) + { + return input_shape; + } + } } diff --git a/src/TensorFlowNET.Keras/Layers/Activation/SELU.cs b/src/TensorFlowNET.Keras/Layers/Activation/SELU.cs index 388302dac..b12d7deec 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/SELU.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/SELU.cs @@ -15,7 +15,7 @@ public class SELU : Layer { public SELU ( LayerArgs args ) : base(args) { // SELU has no arguments } - protected override void build ( Tensors inputs ) { + public override void build(Shape input_shape) { if ( alpha < 0f ) { throw new ValueError("Alpha must be a number greater than 0."); } diff --git a/src/TensorFlowNET.Keras/Layers/Attention/Attention.cs b/src/TensorFlowNET.Keras/Layers/Attention/Attention.cs index 51a40b58c..6f6dd7e85 100644 --- a/src/TensorFlowNET.Keras/Layers/Attention/Attention.cs +++ b/src/TensorFlowNET.Keras/Layers/Attention/Attention.cs @@ -90,9 +90,10 @@ public Attention(AttentionArgs args) : base(args) }.Contains(this.score_mode)) throw new ValueError("Received: score_mode={score_mode}. Acceptable values are: [\"dot\", \"concat\"]"); } - + // Creates variable when `use_scale` is True or `score_mode` is `concat`. - protected override void build(Tensors inputs) { + public override void build(Shape input_shape) + { if (this.use_scale) this.scale = this.add_weight(name: "scale", shape: 1, @@ -110,7 +111,7 @@ protected override void build(Tensors inputs) { trainable: true); else this.concat_score_weight = null; - base.build(inputs); + base.build(input_shape); } /// diff --git a/src/TensorFlowNET.Keras/Layers/Convolution/Conv2DTranspose.cs b/src/TensorFlowNET.Keras/Layers/Convolution/Conv2DTranspose.cs index 9ef4db182..e0a337caa 100644 --- a/src/TensorFlowNET.Keras/Layers/Convolution/Conv2DTranspose.cs +++ b/src/TensorFlowNET.Keras/Layers/Convolution/Conv2DTranspose.cs @@ -29,9 +29,8 @@ public Conv2DTranspose(Conv2DArgs args) : base(args) } - protected override void build(Tensors inputs) + public override void build(Shape input_shape) { - var input_shape = inputs.shape; if (len(input_shape) != 4) throw new ValueError($"Inputs should have rank 4. Received input shape: {input_shape}"); @@ -43,14 +42,12 @@ protected override void build(Tensors inputs) shape: kernel_shape, initializer: kernel_initializer, regularizer: kernel_regularizer, - trainable: true, - dtype: inputs.dtype); + trainable: true); if (use_bias) bias = add_weight(name: "bias", shape: filters, initializer: bias_initializer, - trainable: true, - dtype: inputs.dtype); + trainable: true); built = true; } diff --git a/src/TensorFlowNET.Keras/Layers/Convolution/Convolutional.cs b/src/TensorFlowNET.Keras/Layers/Convolution/Convolutional.cs index 5ac2dd003..912a429b7 100644 --- a/src/TensorFlowNET.Keras/Layers/Convolution/Convolutional.cs +++ b/src/TensorFlowNET.Keras/Layers/Convolution/Convolutional.cs @@ -57,9 +57,8 @@ public Convolutional(ConvolutionalArgs args) : base(args) _tf_data_format = conv_utils.convert_data_format(data_format, rank + 2); } - protected override void build(Tensors inputs) + public override void build(Shape input_shape) { - Shape input_shape = inputs.shape; int channel_axis = data_format == "channels_first" ? 1 : -1; var input_channel = channel_axis < 0 ? input_shape.dims[input_shape.ndim + channel_axis] : diff --git a/src/TensorFlowNET.Keras/Layers/Core/Dense.cs b/src/TensorFlowNET.Keras/Layers/Core/Dense.cs index f3956811f..e4c227456 100644 --- a/src/TensorFlowNET.Keras/Layers/Core/Dense.cs +++ b/src/TensorFlowNET.Keras/Layers/Core/Dense.cs @@ -41,9 +41,8 @@ public Dense(DenseArgs args) : this.inputSpec = new InputSpec(min_ndim: 2); } - protected override void build(Tensors inputs) + public override void build(Shape input_shape) { - Shape input_shape = inputs.shape; var last_dim = input_shape.dims.Last(); var axes = new Dictionary(); axes[-1] = (int)last_dim; diff --git a/src/TensorFlowNET.Keras/Layers/Core/EinsumDense.cs b/src/TensorFlowNET.Keras/Layers/Core/EinsumDense.cs index 2bd987a7c..0f387570b 100644 --- a/src/TensorFlowNET.Keras/Layers/Core/EinsumDense.cs +++ b/src/TensorFlowNET.Keras/Layers/Core/EinsumDense.cs @@ -119,9 +119,8 @@ public EinsumDense(EinsumDenseArgs args) : base(args) this.bias_constraint = args.BiasConstraint; } - protected override void build(Tensors inputs) + public override void build(Shape input_shape) { - var input_shape = inputs.shape; var shape_data = _analyze_einsum_string(this.equation, this.bias_axes, input_shape, this.partial_output_shape); var kernel_shape = shape_data.Item1; var bias_shape = shape_data.Item2; @@ -141,7 +140,7 @@ protected override void build(Tensors inputs) trainable: true); else this.bias = null; - base.build(inputs); + base.build(input_shape); } public override Shape ComputeOutputShape(Shape input_shape) diff --git a/src/TensorFlowNET.Keras/Layers/Core/Embedding.cs b/src/TensorFlowNET.Keras/Layers/Core/Embedding.cs index f16fcfa6f..79f4e5ce9 100644 --- a/src/TensorFlowNET.Keras/Layers/Core/Embedding.cs +++ b/src/TensorFlowNET.Keras/Layers/Core/Embedding.cs @@ -54,7 +54,7 @@ public Embedding(EmbeddingArgs args) SupportsMasking = mask_zero; } - protected override void build(Tensors inputs) + public override void build(Shape input_shape) { tf.Context.eager_mode(); embeddings = add_weight(shape: (input_dim, output_dim), diff --git a/src/TensorFlowNET.Keras/Layers/Cropping/Cropping1D.cs b/src/TensorFlowNET.Keras/Layers/Cropping/Cropping1D.cs index cf71e1845..45f5bf0f6 100644 --- a/src/TensorFlowNET.Keras/Layers/Cropping/Cropping1D.cs +++ b/src/TensorFlowNET.Keras/Layers/Cropping/Cropping1D.cs @@ -2,49 +2,61 @@ using Tensorflow.Keras.Engine; namespace Tensorflow.Keras.Layers { - public class Cropping1D : Layer { - CroppingArgs args; - public Cropping1D ( CroppingArgs args ) : base(args) { - this.args = args; - } + public class Cropping1D : Layer + { + CroppingArgs args; + public Cropping1D(CroppingArgs args) : base(args) + { + this.args = args; + } - protected override void build ( Tensors inputs ) { - if ( args.cropping.rank != 1 ) { - // throw an ValueError exception - throw new ValueError(""); - } - else if ( args.cropping.shape[0] > 2 || args.cropping.shape[0] < 1 ) { - throw new ValueError("The `cropping` argument must be a tuple of 2 integers."); - } - built = true; + public override void build(Shape input_shape) + { + if (args.cropping.rank != 1) + { + // throw an ValueError exception + throw new ValueError(""); + } + else if (args.cropping.shape[0] > 2 || args.cropping.shape[0] < 1) + { + throw new ValueError("The `cropping` argument must be a tuple of 2 integers."); } + built = true; + } - protected override Tensors Call ( Tensors inputs, Tensor state = null, bool? training = null ) { - Tensor output = inputs; - if ( output.rank != 3 ) { - // throw an ValueError exception - throw new ValueError("Expected dim=3, found dim=" + output.rank); - } - if ( args.cropping.shape[0] == 1 ) { - int crop_start = args.cropping[0]; - output = output[new Slice(), new Slice(crop_start, ( int ) output.shape[1] - crop_start), new Slice()]; - } - else { - int crop_start = args.cropping[0], crop_end = args.cropping[1]; - output = output[new Slice(), new Slice(crop_start, ( int ) (output.shape[1]) - crop_end), new Slice()]; - } - return output; + protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + { + Tensor output = inputs; + if (output.rank != 3) + { + // throw an ValueError exception + throw new ValueError("Expected dim=3, found dim=" + output.rank); + } + if (args.cropping.shape[0] == 1) + { + int crop_start = args.cropping[0]; + output = output[new Slice(), new Slice(crop_start, (int)output.shape[1] - crop_start), new Slice()]; } + else + { + int crop_start = args.cropping[0], crop_end = args.cropping[1]; + output = output[new Slice(), new Slice(crop_start, (int)(output.shape[1]) - crop_end), new Slice()]; + } + return output; + } - public override Shape ComputeOutputShape ( Shape input_shape ) { - if ( args.cropping.shape[0] == 1 ) { - int crop = args.cropping[0]; - return new Shape(( int ) (input_shape[0]), ( int ) (input_shape[1] - crop * 2), ( int ) (input_shape[2])); - } - else { - int crop_start = args.cropping[0], crop_end = args.cropping[1]; - return new Shape(( int ) (input_shape[0]), ( int ) (input_shape[1] - crop_start - crop_end), ( int ) (input_shape[2])); - } + public override Shape ComputeOutputShape(Shape input_shape) + { + if (args.cropping.shape[0] == 1) + { + int crop = args.cropping[0]; + return new Shape((int)(input_shape[0]), (int)(input_shape[1] - crop * 2), (int)(input_shape[2])); + } + else + { + int crop_start = args.cropping[0], crop_end = args.cropping[1]; + return new Shape((int)(input_shape[0]), (int)(input_shape[1] - crop_start - crop_end), (int)(input_shape[2])); } - } + } + } } diff --git a/src/TensorFlowNET.Keras/Layers/Cropping/Cropping2D.cs b/src/TensorFlowNET.Keras/Layers/Cropping/Cropping2D.cs index 340ba42df..6cb03e1e0 100644 --- a/src/TensorFlowNET.Keras/Layers/Cropping/Cropping2D.cs +++ b/src/TensorFlowNET.Keras/Layers/Cropping/Cropping2D.cs @@ -12,7 +12,7 @@ public class Cropping2D : Layer { public Cropping2D ( Cropping2DArgs args ) : base(args) { this.args = args; } - protected override void build ( Tensors inputs ) { + public override void build(Shape input_shape) { built = true; } protected override Tensors Call ( Tensors inputs, Tensor state = null, bool? training = null ) { diff --git a/src/TensorFlowNET.Keras/Layers/Cropping/Cropping3D.cs b/src/TensorFlowNET.Keras/Layers/Cropping/Cropping3D.cs index df102c1fa..2d6751bf9 100644 --- a/src/TensorFlowNET.Keras/Layers/Cropping/Cropping3D.cs +++ b/src/TensorFlowNET.Keras/Layers/Cropping/Cropping3D.cs @@ -11,7 +11,7 @@ public Cropping3D ( Cropping3DArgs args ) : base(args) { this.args = args; } - protected override void build ( Tensors inputs ) { + public override void build(Shape input_shape) { built = true; } diff --git a/src/TensorFlowNET.Keras/Layers/Merging/Concatenate.cs b/src/TensorFlowNET.Keras/Layers/Merging/Concatenate.cs index 676d5752b..5f8217604 100644 --- a/src/TensorFlowNET.Keras/Layers/Merging/Concatenate.cs +++ b/src/TensorFlowNET.Keras/Layers/Merging/Concatenate.cs @@ -23,7 +23,7 @@ public Concatenate(MergeArgs args) : base(args) this.args = args; } - protected override void build(Tensors inputs) + public override void build(Shape input_shape) { /*var shape_set = new HashSet(); var reduced_inputs_shapes = inputs.Select(x => x.shape).ToArray(); diff --git a/src/TensorFlowNET.Keras/Layers/Merging/Merge.cs b/src/TensorFlowNET.Keras/Layers/Merging/Merge.cs index be8f574ec..0363d58f4 100644 --- a/src/TensorFlowNET.Keras/Layers/Merging/Merge.cs +++ b/src/TensorFlowNET.Keras/Layers/Merging/Merge.cs @@ -14,7 +14,7 @@ public Merge(MergeArgs args) : base(args) } - protected override void build(Tensors inputs) + public override void build(Shape input_shape) { // output_shape = input_shape.dims[1^]; } diff --git a/src/TensorFlowNET.Keras/Layers/Normalization/BatchNormalization.cs b/src/TensorFlowNET.Keras/Layers/Normalization/BatchNormalization.cs index da8e8c037..dac92f812 100644 --- a/src/TensorFlowNET.Keras/Layers/Normalization/BatchNormalization.cs +++ b/src/TensorFlowNET.Keras/Layers/Normalization/BatchNormalization.cs @@ -53,9 +53,8 @@ public BatchNormalization(BatchNormalizationArgs args) : base(args) axis = args.Axis.dims.Select(x => (int)x).ToArray(); } - protected override void build(Tensors inputs) + public override void build(Shape input_shape) { - Shape input_shape = inputs.shape; var ndims = input_shape.ndim; foreach (var (idx, x) in enumerate(axis)) if (x < 0) diff --git a/src/TensorFlowNET.Keras/Layers/Normalization/LayerNormalization.cs b/src/TensorFlowNET.Keras/Layers/Normalization/LayerNormalization.cs index 51c6423c8..5eebd7350 100644 --- a/src/TensorFlowNET.Keras/Layers/Normalization/LayerNormalization.cs +++ b/src/TensorFlowNET.Keras/Layers/Normalization/LayerNormalization.cs @@ -49,9 +49,8 @@ public LayerNormalization(LayerNormalizationArgs args) : base(args) axis = args.Axis.axis; } - protected override void build(Tensors inputs) + public override void build(Shape input_shape) { - Shape input_shape = inputs.shape; var ndims = input_shape.ndim; foreach (var (idx, x) in enumerate(axis)) if (x < 0) diff --git a/src/TensorFlowNET.Keras/Layers/Preprocessing/TextVectorization.cs b/src/TensorFlowNET.Keras/Layers/Preprocessing/TextVectorization.cs index 6d37eaa12..4c52af9ba 100644 --- a/src/TensorFlowNET.Keras/Layers/Preprocessing/TextVectorization.cs +++ b/src/TensorFlowNET.Keras/Layers/Preprocessing/TextVectorization.cs @@ -35,14 +35,14 @@ public override void adapt(IDatasetV2 data, bool reset_state = true) var shape = data.output_shapes[0]; if (shape.ndim == 1) data = data.map(tensor => array_ops.expand_dims(tensor, -1)); - build(data.variant_tensor); + build(data.variant_tensor.shape); var preprocessed_inputs = data.map(_preprocess); _index_lookup_layer.adapt(preprocessed_inputs); } - protected override void build(Tensors inputs) + public override void build(Shape input_shape) { - base.build(inputs); + base.build(input_shape); } Tensors _preprocess(Tensors inputs) diff --git a/src/TensorFlowNET.Keras/Layers/Reshaping/Permute.cs b/src/TensorFlowNET.Keras/Layers/Reshaping/Permute.cs index 08089900a..868506b6b 100644 --- a/src/TensorFlowNET.Keras/Layers/Reshaping/Permute.cs +++ b/src/TensorFlowNET.Keras/Layers/Reshaping/Permute.cs @@ -7,32 +7,39 @@ using Tensorflow.Keras.ArgsDefinition; namespace Tensorflow.Keras.Layers { - public class Permute : Layer { - int[] dims, permute; - public Permute ( PermuteArgs args ) : base(args) { - this.dims = args.dims; + public class Permute : Layer + { + int[] dims, permute; + public Permute(PermuteArgs args) : base(args) + { + this.dims = args.dims; + } + public override void build(Shape input_shape) + { + var rank = input_shape.rank; + if (dims.Length != rank - 1) + { + throw new ValueError("Dimensions must match."); } - protected override void build ( Tensors inputs ) { - var rank = inputs.rank; - if ( dims.Length != rank - 1 ) { - throw new ValueError("Dimensions must match."); - } - permute = new int[inputs.rank]; - dims.CopyTo(permute, 1); - built = true; + permute = new int[input_shape.rank]; + dims.CopyTo(permute, 1); + built = true; + } + protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + { + Tensor outputs = inputs; + return tf.transpose(outputs, new Axis(permute)); + } + public override Shape ComputeOutputShape(Shape input_shape) + { + Shape output_shape = new Shape(input_shape.dims); + for (int i = 0; i < dims.Length; i += 1) + { + var d = dims[i]; + var target_dim = input_shape[d]; + output_shape[i + 1] = target_dim; } - protected override Tensors Call ( Tensors inputs, Tensor state = null, bool? training = null ) { - Tensor outputs = inputs; - return tf.transpose(outputs, new Axis(permute)); - } - public override Shape ComputeOutputShape ( Shape input_shape ) { - Shape output_shape = new Shape(input_shape.dims); - for ( int i = 0; i < dims.Length; i += 1 ) { - var d = dims[i]; - var target_dim = input_shape[d]; - output_shape[i + 1] = target_dim; - } - return output_shape; - } - } + return output_shape; + } + } } diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs index 58b700fe6..c8366ff48 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs @@ -15,9 +15,8 @@ public SimpleRNN(SimpleRNNArgs args) : base(args) this.args = args; } - protected override void build(Tensors inputs) + public override void build(Shape input_shape) { - var input_shape = inputs.shape; var input_dim = input_shape[-1]; kernel = add_weight("kernel", (input_shape[-1], args.Units), diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs index de50c3618..10b28e76a 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs @@ -13,7 +13,7 @@ public SimpleRNNCell(SimpleRNNArgs args) : base(args) } - protected override void build(Tensors inputs) + public override void build(Shape input_shape) { } From 290c162eb25103e0166fe4dedd5400d4be2b395d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Dec 2022 14:54:42 +0000 Subject: [PATCH 08/52] Bump Newtonsoft.Json from 13.0.1 to 13.0.2 in /src/TensorFlowNET.Keras Bumps [Newtonsoft.Json](https://github.com/JamesNK/Newtonsoft.Json) from 13.0.1 to 13.0.2. - [Release notes](https://github.com/JamesNK/Newtonsoft.Json/releases) - [Commits](https://github.com/JamesNK/Newtonsoft.Json/compare/13.0.1...13.0.2) --- updated-dependencies: - dependency-name: Newtonsoft.Json dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- src/TensorFlowNET.Keras/Tensorflow.Keras.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj index 23c996208..647601a77 100644 --- a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj +++ b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj @@ -72,7 +72,7 @@ Keras is an API designed for human beings, not machines. Keras follows best prac - + From 2584b684e4ba978d43da0ec645d9b238871870fe Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 30 Dec 2022 09:18:55 -0600 Subject: [PATCH 09/52] Fix conv_net.load_weights #956 --- .../Engine/Layer.Layers.cs | 2 +- src/TensorFlowNET.Keras/Engine/Model.Build.cs | 28 ++++ src/python/subclassing.py | 154 ++++++++++++++++++ 3 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 src/TensorFlowNET.Keras/Engine/Model.Build.cs create mode 100644 src/python/subclassing.py diff --git a/src/TensorFlowNET.Keras/Engine/Layer.Layers.cs b/src/TensorFlowNET.Keras/Engine/Layer.Layers.cs index f38750c25..488c55cb7 100644 --- a/src/TensorFlowNET.Keras/Engine/Layer.Layers.cs +++ b/src/TensorFlowNET.Keras/Engine/Layer.Layers.cs @@ -10,7 +10,7 @@ public partial class Layer protected void StackLayers(params ILayer[] layers) { - _layers.AddRange(layers); + _self_tracked_trackables.AddRange(layers); } public virtual Shape ComputeOutputShape(Shape input_shape) diff --git a/src/TensorFlowNET.Keras/Engine/Model.Build.cs b/src/TensorFlowNET.Keras/Engine/Model.Build.cs new file mode 100644 index 000000000..1e0a880a6 --- /dev/null +++ b/src/TensorFlowNET.Keras/Engine/Model.Build.cs @@ -0,0 +1,28 @@ +using System; +using System.Linq; +using Tensorflow.Graphs; +using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.Losses; +using Tensorflow.Keras.Optimizers; +using static Tensorflow.Binding; +using static Tensorflow.KerasApi; + +namespace Tensorflow.Keras.Engine +{ + public partial class Model + { + public override void build(Shape input_shape) + { + var graph = tf.executing_eagerly() ? new FuncGraph("build_graph") : keras.backend.get_graph(); + + graph.as_default(); + + var x = tf.placeholder(DType, input_shape); + Call(x, training: false); + + graph.Exit(); + + base.build(input_shape); + } + } +} diff --git a/src/python/subclassing.py b/src/python/subclassing.py new file mode 100644 index 000000000..fe2674ec9 --- /dev/null +++ b/src/python/subclassing.py @@ -0,0 +1,154 @@ +from __future__ import absolute_import, division, print_function + +import tensorflow as tf +from tensorflow.keras import Model, layers +import numpy as np + +# MNIST dataset parameters. +num_classes = 10 # total classes (0-9 digits). + +# Training parameters. +learning_rate = 0.001 +training_steps = 100 +batch_size = 128 +display_step = 10 + +# Network parameters. +conv1_filters = 32 # number of filters for 1st conv layer. +conv2_filters = 64 # number of filters for 2nd conv layer. +fc1_units = 1024 # number of neurons for 1st fully-connected layer. + +# Prepare MNIST data. +from tensorflow.keras.datasets import mnist +(x_train, y_train), (x_test, y_test) = mnist.load_data() +# Convert to float32. +x_train, x_test = np.array(x_train, np.float32), np.array(x_test, np.float32) +# Normalize images value from [0, 255] to [0, 1]. +x_train, x_test = x_train / 255., x_test / 255. + +# Use tf.data API to shuffle and batch data. +train_data = tf.data.Dataset.from_tensor_slices((x_train, y_train)) +train_data = train_data.repeat().shuffle(5000).batch(batch_size).prefetch(1) + +# Create TF Model. +class ConvNet(Model): + # Set layers. + def __init__(self): + super(ConvNet, self).__init__() + # Convolution Layer with 32 filters and a kernel size of 5. + self.conv1 = layers.Conv2D(32, kernel_size=5, activation=tf.nn.relu) + # Max Pooling (down-sampling) with kernel size of 2 and strides of 2. + self.maxpool1 = layers.MaxPool2D(2, strides=2) + + # Convolution Layer with 64 filters and a kernel size of 3. + self.conv2 = layers.Conv2D(64, kernel_size=3, activation=tf.nn.relu) + # Max Pooling (down-sampling) with kernel size of 2 and strides of 2. + self.maxpool2 = layers.MaxPool2D(2, strides=2) + + # Flatten the data to a 1-D vector for the fully connected layer. + self.flatten = layers.Flatten() + + # Fully connected layer. + self.fc1 = layers.Dense(1024) + # Apply Dropout (if is_training is False, dropout is not applied). + self.dropout = layers.Dropout(rate=0.5) + + # Output layer, class prediction. + self.out = layers.Dense(num_classes) + + # Set forward pass. + def call(self, x, is_training=False): + x = tf.reshape(x, [-1, 28, 28, 1]) + x = self.conv1(x) + x = self.maxpool1(x) + x = self.conv2(x) + x = self.maxpool2(x) + x = self.flatten(x) + x = self.fc1(x) + x = self.dropout(x, training=is_training) + x = self.out(x) + if not is_training: + # tf cross entropy expect logits without softmax, so only + # apply softmax when not training. + x = tf.nn.softmax(x) + return x +''' +# Build neural network model. +conv_net = ConvNet() + +# Cross-Entropy Loss. +# Note that this will apply 'softmax' to the logits. +def cross_entropy_loss(x, y): + # Convert labels to int 64 for tf cross-entropy function. + y = tf.cast(y, tf.int64) + # Apply softmax to logits and compute cross-entropy. + loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=x) + # Average loss across the batch. + return tf.reduce_mean(loss) + +# Accuracy metric. +def accuracy(y_pred, y_true): + # Predicted class is the index of highest score in prediction vector (i.e. argmax). + correct_prediction = tf.equal(tf.argmax(y_pred, 1), tf.cast(y_true, tf.int64)) + return tf.reduce_mean(tf.cast(correct_prediction, tf.float32), axis=-1) + +# Stochastic gradient descent optimizer. +optimizer = tf.optimizers.Adam(learning_rate) + +# Optimization process. +def run_optimization(x, y): + # Wrap computation inside a GradientTape for automatic differentiation. + with tf.GradientTape() as g: + # Forward pass. + pred = conv_net(x, is_training=True) + # Compute loss. + loss = cross_entropy_loss(pred, y) + + # Variables to update, i.e. trainable variables. + trainable_variables = conv_net.trainable_variables + + # Compute gradients. + gradients = g.gradient(loss, trainable_variables) + + # Update W and b following gradients. + optimizer.apply_gradients(zip(gradients, trainable_variables)) + +# Run training for the given number of steps. + +for step, (batch_x, batch_y) in enumerate(train_data.take(training_steps), 1): + # Run the optimization to update W and b values. + run_optimization(batch_x, batch_y) + + if step % display_step == 0: + pred = conv_net(batch_x) + loss = cross_entropy_loss(pred, batch_y) + acc = accuracy(pred, batch_y) + print("step: %i, loss: %f, accuracy: %f" % (step, loss, acc)) + +# Test model on validation set. +pred = conv_net(x_test) +print("Test Accuracy: %f" % accuracy(pred, y_test)) + +conv_net.save_weights('weights.h5') +''' + +conv_net = ConvNet() +conv_net.build(x_test.shape) +conv_net.load_weights('weights.h5') +# Test model on validation set. +pred = conv_net(x_test) +# print("Test Accuracy: %f" % accuracy(pred, y_test)) + +# Visualize predictions. +import matplotlib.pyplot as plt + +# Predict 5 images from validation set. +n_images = 5 +test_images = x_test[:n_images] +predictions = conv_net(test_images) + +# Display image and model prediction. +for i in range(n_images): + plt.imshow(np.reshape(test_images[i], [28, 28]), cmap='gray') + plt.show() + print("Model prediction: %i" % np.argmax(predictions.numpy()[i])) \ No newline at end of file From a432807c2351a961a0a2007ae18bbdb28c19503c Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 30 Dec 2022 12:43:36 -0600 Subject: [PATCH 10/52] Remove _layers in Layer. --- src/TensorFlowNET.Core/Keras/Layers/ILayer.cs | 8 ++++---- src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs | 8 ++++---- src/TensorFlowNET.Keras/Engine/Functional.GetConfig.cs | 4 ++-- src/TensorFlowNET.Keras/Engine/Functional.cs | 7 +------ src/TensorFlowNET.Keras/Engine/Layer.Layers.cs | 3 +-- src/TensorFlowNET.Keras/Engine/Layer.cs | 8 ++++---- src/TensorFlowNET.Keras/Engine/Model.Train.cs | 2 +- src/TensorFlowNET.Keras/Engine/Model.cs | 8 ++++---- src/TensorFlowNET.Keras/Engine/Sequential.cs | 8 ++++---- src/TensorFlowNET.Keras/Saving/hdf5_format.cs | 4 ++-- src/TensorFlowNET.Keras/Utils/layer_utils.cs | 6 +++--- 11 files changed, 30 insertions(+), 36 deletions(-) diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs index 271fece08..f77b4a86d 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs @@ -13,10 +13,10 @@ public interface ILayer List InboundNodes { get; } List OutboundNodes { get; } Tensors Apply(Tensors inputs, Tensor state = null, bool is_training = false); - List trainable_variables { get; } - List trainable_weights { get; } - List non_trainable_weights { get; } - Shape output_shape { get; } + List TrainableVariables { get; } + List TrainableWeights { get; } + List NonTrainableWeights { get; } + Shape OutputShape { get; } Shape BatchInputShape { get; } TF_DataType DType { get; } int count_params(); diff --git a/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs b/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs index 041268b70..04fdc7e57 100644 --- a/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs +++ b/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs @@ -67,11 +67,11 @@ public abstract class RnnCell : ILayer, RNNArgs.IRnnArgCell public bool Trainable => throw new NotImplementedException(); - public List trainable_variables => throw new NotImplementedException(); - public List trainable_weights => throw new NotImplementedException(); - public List non_trainable_weights => throw new NotImplementedException(); + public List TrainableVariables => throw new NotImplementedException(); + public List TrainableWeights => throw new NotImplementedException(); + public List NonTrainableWeights => throw new NotImplementedException(); - public Shape output_shape => throw new NotImplementedException(); + public Shape OutputShape => throw new NotImplementedException(); public Shape BatchInputShape => throw new NotImplementedException(); diff --git a/src/TensorFlowNET.Keras/Engine/Functional.GetConfig.cs b/src/TensorFlowNET.Keras/Engine/Functional.GetConfig.cs index 6615810be..23c40fbff 100644 --- a/src/TensorFlowNET.Keras/Engine/Functional.GetConfig.cs +++ b/src/TensorFlowNET.Keras/Engine/Functional.GetConfig.cs @@ -27,7 +27,7 @@ ModelConfig get_network_config() }; var node_conversion_map = new Dictionary(); - foreach (var layer in _layers) + foreach (var layer in _self_tracked_trackables) { var kept_nodes = _should_skip_first_node(layer) ? 1 : 0; foreach (var (original_node_index, node) in enumerate(layer.InboundNodes)) @@ -42,7 +42,7 @@ ModelConfig get_network_config() } var layer_configs = new List(); - foreach (var layer in _layers) + foreach (var layer in _self_tracked_trackables) { var filtered_inbound_nodes = new List(); foreach (var (original_node_index, node) in enumerate(layer.InboundNodes)) diff --git a/src/TensorFlowNET.Keras/Engine/Functional.cs b/src/TensorFlowNET.Keras/Engine/Functional.cs index def842c32..09a31b948 100644 --- a/src/TensorFlowNET.Keras/Engine/Functional.cs +++ b/src/TensorFlowNET.Keras/Engine/Functional.cs @@ -65,13 +65,8 @@ protected void _init_graph_network(Tensors inputs, Tensors outputs) } // Keep track of the network's nodes and layers. - var (nodes, nodes_by_depth, layers, _) = MapGraphNetwork(inputs, outputs); + (NetworkNodes, NodesByDepth, _self_tracked_trackables, _) = MapGraphNetwork(inputs, outputs); - NetworkNodes = nodes; - NodesByDepth = nodes_by_depth; - if (_layers.Count == 0) - _layers = layers; - _self_tracked_trackables = layers; // Build self.input_names and self.output_names. _set_output_names(); diff --git a/src/TensorFlowNET.Keras/Engine/Layer.Layers.cs b/src/TensorFlowNET.Keras/Engine/Layer.Layers.cs index 488c55cb7..a2d212cb3 100644 --- a/src/TensorFlowNET.Keras/Engine/Layer.Layers.cs +++ b/src/TensorFlowNET.Keras/Engine/Layer.Layers.cs @@ -5,8 +5,7 @@ namespace Tensorflow.Keras.Engine { public partial class Layer { - protected List _layers = new List(); - public virtual List Layers => _layers; + public virtual List Layers => _self_tracked_trackables; protected void StackLayers(params ILayer[] layers) { diff --git a/src/TensorFlowNET.Keras/Engine/Layer.cs b/src/TensorFlowNET.Keras/Engine/Layer.cs index d417fa44b..ba40b1a22 100644 --- a/src/TensorFlowNET.Keras/Engine/Layer.cs +++ b/src/TensorFlowNET.Keras/Engine/Layer.cs @@ -63,7 +63,7 @@ public abstract partial class Layer : AutoTrackable, ILayer public bool SupportsMasking { get; set; } protected List _trainable_weights; - public virtual List trainable_variables => _trainable_weights; + public virtual List TrainableVariables => _trainable_weights; protected List _non_trainable_weights; public List non_trainable_variables => _non_trainable_weights; @@ -88,7 +88,7 @@ public abstract partial class Layer : AutoTrackable, ILayer public CallContext CallContext => callContext.Value; public Tensor[] input => inboundNodes[0].input_tensors; public Dictionary> NodesByDepth { get; set; } - public Shape output_shape => inboundNodes[0].Outputs.shape; + public Shape OutputShape => inboundNodes[0].Outputs.shape; protected List _self_tracked_trackables; public Layer(LayerArgs args) @@ -250,7 +250,7 @@ public int count_params() return layer_utils.count_params(this, weights); return 0; } - List ILayer.trainable_weights + List ILayer.TrainableWeights { get { @@ -258,7 +258,7 @@ List ILayer.trainable_weights } } - List ILayer.non_trainable_weights + List ILayer.NonTrainableWeights { get { diff --git a/src/TensorFlowNET.Keras/Engine/Model.Train.cs b/src/TensorFlowNET.Keras/Engine/Model.Train.cs index 31e89c573..f2ff68e97 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Train.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Train.cs @@ -34,7 +34,7 @@ public partial class Model // self.optimizer.apply_gradients(zip(gradients, trainable_variables)) // The _minimize call does a few extra steps unnecessary in most cases, // such as loss scaling and gradient clipping. - _minimize(tape, optimizer, loss, trainable_variables); + _minimize(tape, optimizer, loss, TrainableVariables); compiled_metrics.update_state(y, y_pred); return metrics.Select(x => (x.Name, x.result())).ToList(); diff --git a/src/TensorFlowNET.Keras/Engine/Model.cs b/src/TensorFlowNET.Keras/Engine/Model.cs index baf68229a..162d06c57 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.cs @@ -74,7 +74,7 @@ void _init_batch_counters() public override List Layers => _flatten_layers(recursive: false, include_self: false).ToList(); - public override List trainable_variables + public override List TrainableVariables { get { @@ -88,13 +88,13 @@ public override List trainable_variables foreach (var trackable_obj in _self_tracked_trackables) { if (trackable_obj.Trainable) - variables.AddRange(trackable_obj.trainable_variables); + variables.AddRange(trackable_obj.TrainableVariables); } - foreach (var layer in _layers) + foreach (var layer in _self_tracked_trackables) { if (layer.Trainable) - variables.AddRange(layer.trainable_variables); + variables.AddRange(layer.TrainableVariables); } // variables.AddRange(_trainable_weights); diff --git a/src/TensorFlowNET.Keras/Engine/Sequential.cs b/src/TensorFlowNET.Keras/Engine/Sequential.cs index 47e6c3f77..681ab2f04 100644 --- a/src/TensorFlowNET.Keras/Engine/Sequential.cs +++ b/src/TensorFlowNET.Keras/Engine/Sequential.cs @@ -75,7 +75,7 @@ public void add(ILayer layer) { built = false; var set_inputs = false; - if (_layers.Count == 0) + if (_self_tracked_trackables.Count == 0) { if (layer is InputLayer) { @@ -128,7 +128,7 @@ public void add(ILayer layer) void _handle_deferred_layer_dependencies(params ILayer[] layers) { - _layers.AddRange(layers); + _self_tracked_trackables.AddRange(layers); } protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) @@ -156,12 +156,12 @@ void _build_graph_network_for_inferred_shape(Shape input_shape, TF_DataType inpu ops.init_scope(); var inputs = keras.Input(batch_input_shape: input_shape, dtype: input_dtype, - name: $"{_layers[0].Name}_input"); + name: $"{_self_tracked_trackables[0].Name}_input"); Tensors layer_input = inputs; Tensors layer_output = null; Tensors outputs = null; List created_nodes = new List(); - foreach (var layer in _layers) + foreach (var layer in _self_tracked_trackables) { clear_previously_created_nodes(layer, _created_nodes); layer_output = layer.Apply(layer_input); diff --git a/src/TensorFlowNET.Keras/Saving/hdf5_format.cs b/src/TensorFlowNET.Keras/Saving/hdf5_format.cs index a3705dfba..b04391be9 100644 --- a/src/TensorFlowNET.Keras/Saving/hdf5_format.cs +++ b/src/TensorFlowNET.Keras/Saving/hdf5_format.cs @@ -338,8 +338,8 @@ public static void load_attributes_from_hdf5_group(long filepath = -1, Dictionar public static List _legacy_weights(ILayer layer) { - var weights = layer.trainable_weights.Select(x => x).ToList(); - weights.AddRange(layer.non_trainable_weights); + var weights = layer.TrainableWeights.Select(x => x).ToList(); + weights.AddRange(layer.NonTrainableWeights); return weights; } } diff --git a/src/TensorFlowNET.Keras/Utils/layer_utils.cs b/src/TensorFlowNET.Keras/Utils/layer_utils.cs index 998086f68..3c38a6d1b 100644 --- a/src/TensorFlowNET.Keras/Utils/layer_utils.cs +++ b/src/TensorFlowNET.Keras/Utils/layer_utils.cs @@ -103,7 +103,7 @@ public static void print_summary(Model model, int line_length = -1, float[] posi print(string.Join("", range(line_length).Select(x => "_"))); } - var trainable_count = count_params(model, model.trainable_variables); + var trainable_count = count_params(model, model.TrainableVariables); var non_trainable_count = count_params(model, model.non_trainable_variables); print($"Total params: {trainable_count + non_trainable_count}"); @@ -137,7 +137,7 @@ static void print_layer_summary(ILayer layer, int[] positions) var fields = new string[] { $"{name} ({layer.GetType().Name})", - $"{layer.output_shape}", + $"{layer.OutputShape}", $"{layer.count_params()}" }; @@ -164,7 +164,7 @@ static void print_layer_summary_with_connections(ILayer layer, int[] positions, var fields = new string[] { $"{name}({layer.GetType().Name})", - $"{layer.output_shape}", + $"{layer.OutputShape}", $"{layer.count_params()}", first_connection }; From 0f7bf4d6a6ce16bab3d15d686918871499d0f05d Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sun, 1 Jan 2023 17:06:07 -0600 Subject: [PATCH 11/52] Fix TransferLearning.Test null bug. --- src/python/subclassing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/subclassing.py b/src/python/subclassing.py index fe2674ec9..bccbef292 100644 --- a/src/python/subclassing.py +++ b/src/python/subclassing.py @@ -65,7 +65,7 @@ def call(self, x, is_training=False): x = self.maxpool2(x) x = self.flatten(x) x = self.fc1(x) - x = self.dropout(x, training=is_training) + x = self.dropout(x) x = self.out(x) if not is_training: # tf cross entropy expect logits without softmax, so only From 321ddfc13ec7c91d2b8e4fea0ad9a7662dd30899 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sun, 15 Jan 2023 21:18:05 -0600 Subject: [PATCH 12/52] Fix Model.build. --- src/TensorFlowNET.Console/SimpleRnnTest.cs | 20 ++++++-------- src/TensorFlowNET.Core/Keras/Layers/ILayer.cs | 1 + .../Keras/Layers/ILayersApi.cs | 4 ++- .../Operations/Initializers/Orthogonal.cs | 22 +++++++++++++++- .../Operations/NnOps/RNNCell.cs | 5 ++++ src/TensorFlowNET.Core/tensorflow.cs | 5 ++++ src/TensorFlowNET.Keras/Engine/Functional.cs | 7 ++++- src/TensorFlowNET.Keras/Engine/Model.Build.cs | 9 ++++--- src/TensorFlowNET.Keras/Engine/Sequential.cs | 8 +----- src/TensorFlowNET.Keras/Layers/LayersApi.cs | 10 ++++--- src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs | 10 ++++++- .../Layers/Rnn/SimpleRNN.cs | 14 +--------- .../Layers/Rnn/SimpleRNNCell.cs | 26 +++++++++++++++++-- .../Layers/LayersTest.cs | 9 ++++--- .../Tensorflow.Binding.UnitTest.csproj | 2 +- 15 files changed, 104 insertions(+), 48 deletions(-) diff --git a/src/TensorFlowNET.Console/SimpleRnnTest.cs b/src/TensorFlowNET.Console/SimpleRnnTest.cs index b61cee9c8..da1245172 100644 --- a/src/TensorFlowNET.Console/SimpleRnnTest.cs +++ b/src/TensorFlowNET.Console/SimpleRnnTest.cs @@ -12,20 +12,16 @@ public class SimpleRnnTest { public void Run() { - tf.keras = new KerasInterface(); - var inputs = np.random.random((32, 10, 8)).astype(np.float32); - var simple_rnn = tf.keras.layers.SimpleRNN(4); - var output = simple_rnn.Apply(inputs); // The output has shape `[32, 4]`. - if (output.shape == (32, 4)) - { + tf.UseKeras(); + var inputs = np.random.random((6, 10, 8)).astype(np.float32); + //var simple_rnn = tf.keras.layers.SimpleRNN(4); + //var output = simple_rnn.Apply(inputs); // The output has shape `[32, 4]`. - } - /*simple_rnn = tf.keras.layers.SimpleRNN( - 4, return_sequences = True, return_state = True) + var simple_rnn = tf.keras.layers.SimpleRNN(4, return_sequences: true, return_state: true); - # whole_sequence_output has shape `[32, 10, 4]`. - # final_state has shape `[32, 4]`. - whole_sequence_output, final_state = simple_rnn(inputs)*/ + // whole_sequence_output has shape `[32, 10, 4]`. + // final_state has shape `[32, 4]`. + var (whole_sequence_output, final_state) = simple_rnn.Apply(inputs); } } } diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs index f77b4a86d..1ec4a2c6e 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs @@ -9,6 +9,7 @@ public interface ILayer string Name { get; } bool Trainable { get; } bool Built { get; } + void build(Shape input_shape); List Layers { get; } List InboundNodes { get; } List OutboundNodes { get; } diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs index 3f4d1ed8e..525bfd354 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs @@ -163,7 +163,9 @@ public ILayer SimpleRNN(int units, string activation = "tanh", string kernel_initializer = "glorot_uniform", string recurrent_initializer = "orthogonal", - string bias_initializer = "zeros"); + string bias_initializer = "zeros", + bool return_sequences = false, + bool return_state = false); public ILayer Subtract(); } diff --git a/src/TensorFlowNET.Core/Operations/Initializers/Orthogonal.cs b/src/TensorFlowNET.Core/Operations/Initializers/Orthogonal.cs index 254a7ee7b..90f3f93c3 100644 --- a/src/TensorFlowNET.Core/Operations/Initializers/Orthogonal.cs +++ b/src/TensorFlowNET.Core/Operations/Initializers/Orthogonal.cs @@ -1,12 +1,32 @@ using System; +using System.Linq; +using static Tensorflow.TensorShapeProto.Types; namespace Tensorflow.Operations.Initializers { public class Orthogonal : IInitializer { + float _gain = 0f; + + public Orthogonal(float gain = 1.0f, int? seed = null) + { + + } + public Tensor Apply(InitializerArgs args) { - throw new NotImplementedException(); + return _generate_init_val(args.Shape, args.DType); + } + + private Tensor _generate_init_val(Shape shape, TF_DataType dtype) + { + var num_rows = 1L; + foreach (var dim in shape.dims.Take(shape.ndim - 1)) + num_rows *= dim; + var num_cols = shape.dims.Last(); + var flat_shape = (Math.Max(num_cols, num_rows), Math.Min(num_cols, num_rows)); + + throw new NotImplementedException(""); } } } diff --git a/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs b/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs index 04fdc7e57..d63d0311b 100644 --- a/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs +++ b/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs @@ -147,5 +147,10 @@ public LayerArgs get_config() { throw new NotImplementedException(); } + + public void build(Shape input_shape) + { + throw new NotImplementedException(); + } } } diff --git a/src/TensorFlowNET.Core/tensorflow.cs b/src/TensorFlowNET.Core/tensorflow.cs index e02723b7c..35762be12 100644 --- a/src/TensorFlowNET.Core/tensorflow.cs +++ b/src/TensorFlowNET.Core/tensorflow.cs @@ -65,6 +65,11 @@ public tensorflow() InitGradientEnvironment(); } + public void UseKeras() where T : IKerasApi, new() + { + keras = new T(); + } + public string VERSION => c_api.StringPiece(c_api.TF_Version()); private void InitGradientEnvironment() diff --git a/src/TensorFlowNET.Keras/Engine/Functional.cs b/src/TensorFlowNET.Keras/Engine/Functional.cs index 09a31b948..d10ed214a 100644 --- a/src/TensorFlowNET.Keras/Engine/Functional.cs +++ b/src/TensorFlowNET.Keras/Engine/Functional.cs @@ -65,7 +65,12 @@ protected void _init_graph_network(Tensors inputs, Tensors outputs) } // Keep track of the network's nodes and layers. - (NetworkNodes, NodesByDepth, _self_tracked_trackables, _) = MapGraphNetwork(inputs, outputs); + (NetworkNodes, NodesByDepth, var layers, _) = MapGraphNetwork(inputs, outputs); + + if (!_self_tracked_trackables.Any()) + { + _self_tracked_trackables = layers; + } // Build self.input_names and self.output_names. _set_output_names(); diff --git a/src/TensorFlowNET.Keras/Engine/Model.Build.cs b/src/TensorFlowNET.Keras/Engine/Model.Build.cs index 1e0a880a6..a51b94348 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Build.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Build.cs @@ -1,9 +1,6 @@ using System; using System.Linq; using Tensorflow.Graphs; -using Tensorflow.Keras.ArgsDefinition; -using Tensorflow.Keras.Losses; -using Tensorflow.Keras.Optimizers; using static Tensorflow.Binding; using static Tensorflow.KerasApi; @@ -13,6 +10,12 @@ public partial class Model { public override void build(Shape input_shape) { + if (this is Functional || this is Sequential) + { + base.build(input_shape); + return; + } + var graph = tf.executing_eagerly() ? new FuncGraph("build_graph") : keras.backend.get_graph(); graph.as_default(); diff --git a/src/TensorFlowNET.Keras/Engine/Sequential.cs b/src/TensorFlowNET.Keras/Engine/Sequential.cs index 681ab2f04..b4d1ecfef 100644 --- a/src/TensorFlowNET.Keras/Engine/Sequential.cs +++ b/src/TensorFlowNET.Keras/Engine/Sequential.cs @@ -122,15 +122,9 @@ public void add(ILayer layer) else { _self_tracked_trackables.add(layer); - _handle_deferred_layer_dependencies(layer); } } - void _handle_deferred_layer_dependencies(params ILayer[] layers) - { - _self_tracked_trackables.AddRange(layers); - } - protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) { if (!_has_explicit_input_shape) @@ -156,7 +150,7 @@ void _build_graph_network_for_inferred_shape(Shape input_shape, TF_DataType inpu ops.init_scope(); var inputs = keras.Input(batch_input_shape: input_shape, dtype: input_dtype, - name: $"{_self_tracked_trackables[0].Name}_input"); + name: _self_tracked_trackables[0].Name.EndsWith("_input") ? _self_tracked_trackables[0].Name : $"{_self_tracked_trackables[0].Name}_input"); Tensors layer_input = inputs; Tensors layer_output = null; Tensors outputs = null; diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index 50c66be70..5c1c8995d 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -658,14 +658,18 @@ public ILayer SimpleRNN(int units, string activation = "tanh", string kernel_initializer = "glorot_uniform", string recurrent_initializer = "orthogonal", - string bias_initializer = "zeros") + string bias_initializer = "zeros", + bool return_sequences = false, + bool return_state = false) => new SimpleRNN(new SimpleRNNArgs { Units = units, Activation = GetActivationByName(activation), KernelInitializer = GetInitializerByName(kernel_initializer), - RecurrentInitializer= GetInitializerByName(recurrent_initializer), - BiasInitializer= GetInitializerByName(bias_initializer) + RecurrentInitializer = GetInitializerByName(recurrent_initializer), + BiasInitializer = GetInitializerByName(bias_initializer), + ReturnSequences = return_sequences, + ReturnState = return_state }); /// diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs index c2b86ae4f..f894f41ff 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs @@ -18,7 +18,7 @@ public class RNN : Layer private int _num_constants = 0; protected IVariableV1 kernel; protected IVariableV1 bias; - + protected ILayer cell; public RNN(RNNArgs args) : base(PreConstruct(args)) { this.args = args; @@ -37,6 +37,14 @@ public RNN(RNNArgs args) : base(PreConstruct(args)) //} } + public override void build(Shape input_shape) + { + if (!cell.Built) + { + cell.build(input_shape); + } + } + private static RNNArgs PreConstruct(RNNArgs args) { if (args.Kwargs == null) diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs index c8366ff48..a3cd002d9 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs @@ -9,22 +9,10 @@ namespace Tensorflow.Keras.Layers.Rnn public class SimpleRNN : RNN { SimpleRNNArgs args; - SimpleRNNCell cell; public SimpleRNN(SimpleRNNArgs args) : base(args) { this.args = args; - } - - public override void build(Shape input_shape) - { - var input_dim = input_shape[-1]; - - kernel = add_weight("kernel", (input_shape[-1], args.Units), - initializer: args.KernelInitializer - //regularizer = self.kernel_regularizer, - //constraint = self.kernel_constraint, - //caching_device = default_caching_device, - ); + cell = new SimpleRNNCell(args); } } } \ No newline at end of file diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs index 10b28e76a..8d696d160 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs @@ -8,14 +8,36 @@ namespace Tensorflow.Keras.Layers.Rnn { public class SimpleRNNCell : Layer { + SimpleRNNArgs args; + IVariableV1 kernel; + IVariableV1 recurrent_kernel; + IVariableV1 bias; + public SimpleRNNCell(SimpleRNNArgs args) : base(args) { - + this.args = args; } public override void build(Shape input_shape) { - + var input_dim = input_shape[-1]; + + kernel = add_weight("kernel", (input_shape[-1], args.Units), + initializer: args.KernelInitializer + ); + + recurrent_kernel = add_weight("recurrent_kernel", (args.Units, args.Units), + initializer: args.RecurrentInitializer + ); + + if (args.UseBias) + { + bias = add_weight("bias", (args.Units), + initializer: args.RecurrentInitializer + ); + } + + built = true; } } } diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs index f4fdf94a5..d4ac4b905 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs @@ -150,10 +150,13 @@ public void EinsumDense() [TestMethod] public void SimpleRNN() { - var inputs = np.random.random((32, 10, 8)).astype(np.float32); - var simple_rnn = keras.layers.SimpleRNN(4); + tf.UseKeras(); + var inputs = np.arange(6 * 10 * 8).reshape((6, 10, 8)).astype(np.float32); + /*var simple_rnn = keras.layers.SimpleRNN(4); var output = simple_rnn.Apply(inputs); - Assert.AreEqual((32, 4), output.shape); + Assert.AreEqual((32, 4), output.shape);*/ + var simple_rnn = tf.keras.layers.SimpleRNN(4, return_sequences: true, return_state: true); + var (whole_sequence_output, final_state) = simple_rnn.Apply(inputs); } [TestMethod] diff --git a/test/TensorFlowNET.UnitTest/Tensorflow.Binding.UnitTest.csproj b/test/TensorFlowNET.UnitTest/Tensorflow.Binding.UnitTest.csproj index 36ff4a3dd..56c212d0e 100644 --- a/test/TensorFlowNET.UnitTest/Tensorflow.Binding.UnitTest.csproj +++ b/test/TensorFlowNET.UnitTest/Tensorflow.Binding.UnitTest.csproj @@ -47,7 +47,7 @@ - + From bd26bbdb31215f84fd0a1003715b871b929f0219 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Mon, 16 Jan 2023 20:39:35 -0600 Subject: [PATCH 13/52] Orthogonal initializer. --- src/TensorFlowNET.Console/Program.cs | 3 + src/TensorFlowNET.Core/APIs/tf.linalg.cs | 6 ++ src/TensorFlowNET.Core/APIs/tf.random.cs | 6 ++ .../Keras/IInitializersApi.cs | 11 +++ src/TensorFlowNET.Core/Keras/IKerasApi.cs | 1 + src/TensorFlowNET.Core/NumPy/NDArrayRender.cs | 1 + .../Operations/Initializers/Orthogonal.cs | 68 +++++++++++++------ .../Operations/gen_array_ops.cs | 3 + .../Operations/gen_random_ops.cs | 12 ++++ .../Operations/linalg_ops.cs | 7 ++ .../Operations/stateless_random_ops.cs | 62 +++++++++++++++++ src/TensorFlowNET.Core/tensorflow.cs | 5 +- .../{Initializers.cs => InitializersApi.cs} | 24 ++++--- src/TensorFlowNET.Keras/KerasInterface.cs | 2 +- .../EagerModeTestBase.cs | 3 + .../InitializerTest.cs | 20 ++++++ 16 files changed, 202 insertions(+), 32 deletions(-) create mode 100644 src/TensorFlowNET.Core/Keras/IInitializersApi.cs create mode 100644 src/TensorFlowNET.Core/Operations/stateless_random_ops.cs rename src/TensorFlowNET.Keras/{Initializers.cs => InitializersApi.cs} (64%) create mode 100644 test/TensorFlowNET.Keras.UnitTest/InitializerTest.cs diff --git a/src/TensorFlowNET.Console/Program.cs b/src/TensorFlowNET.Console/Program.cs index 638fe0a3e..091456f8c 100644 --- a/src/TensorFlowNET.Console/Program.cs +++ b/src/TensorFlowNET.Console/Program.cs @@ -1,4 +1,5 @@ using System; +using Tensorflow.Keras; using static Tensorflow.Binding; namespace Tensorflow @@ -7,6 +8,8 @@ class Program { static void Main(string[] args) { + tf.UseKeras(); + var diag = new Diagnostician(); // diag.Diagnose(@"D:\memory.txt"); diff --git a/src/TensorFlowNET.Core/APIs/tf.linalg.cs b/src/TensorFlowNET.Core/APIs/tf.linalg.cs index 5b79d1384..10c09d994 100644 --- a/src/TensorFlowNET.Core/APIs/tf.linalg.cs +++ b/src/TensorFlowNET.Core/APIs/tf.linalg.cs @@ -58,6 +58,12 @@ public Tensor lstsq(Tensor matrix, Tensor rhs, NDArray l2_regularizer = null, bool fast = true, string name = null) => ops.matrix_solve_ls(matrix, rhs, l2_regularizer: l2_regularizer, fast: fast, name: name); + public Tensors qr(Tensor input, bool full_matrices = true, string name = null) + => ops.qr(input, full_matrices: full_matrices, name: name); + + public Tensor tensor_diag_part(Tensor input, string name = null) + => gen_array_ops.diag_part(input, name: name); + public Tensor tensordot(Tensor x, Tensor y, NDArray axes, string name = null) => math_ops.tensordot(x, y, axes, name: name); } diff --git a/src/TensorFlowNET.Core/APIs/tf.random.cs b/src/TensorFlowNET.Core/APIs/tf.random.cs index 9fbf3924b..4f4962840 100644 --- a/src/TensorFlowNET.Core/APIs/tf.random.cs +++ b/src/TensorFlowNET.Core/APIs/tf.random.cs @@ -39,6 +39,12 @@ public Tensor normal(Shape shape, int? seed = null, string name = null) => random_ops.random_normal(shape, mean, stddev, dtype, seed, name); + public Tensor stateless_normal(Shape shape, + float mean = 0.0f, + float stddev = 1.0f, + TF_DataType dtype = TF_DataType.TF_FLOAT, + string name = null) => stateless_random_ops.stateless_random_normal(shape, mean, stddev, dtype, name: name); + /// /// Outputs random values from a truncated normal distribution. /// diff --git a/src/TensorFlowNET.Core/Keras/IInitializersApi.cs b/src/TensorFlowNET.Core/Keras/IInitializersApi.cs new file mode 100644 index 000000000..ff92040eb --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/IInitializersApi.cs @@ -0,0 +1,11 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras +{ + public interface IInitializersApi + { + IInitializer Orthogonal(float gain = 1.0f, int? seed = null); + } +} diff --git a/src/TensorFlowNET.Core/Keras/IKerasApi.cs b/src/TensorFlowNET.Core/Keras/IKerasApi.cs index 660dcbde7..7f85f02f3 100644 --- a/src/TensorFlowNET.Core/Keras/IKerasApi.cs +++ b/src/TensorFlowNET.Core/Keras/IKerasApi.cs @@ -8,5 +8,6 @@ namespace Tensorflow.Keras public interface IKerasApi { public ILayersApi layers { get; } + public IInitializersApi initializers { get; } } } diff --git a/src/TensorFlowNET.Core/NumPy/NDArrayRender.cs b/src/TensorFlowNET.Core/NumPy/NDArrayRender.cs index 741e25812..02cb5926c 100644 --- a/src/TensorFlowNET.Core/NumPy/NDArrayRender.cs +++ b/src/TensorFlowNET.Core/NumPy/NDArrayRender.cs @@ -109,6 +109,7 @@ static string Render(NDArray array) TF_DataType.TF_INT8 => Render(array.ToArray(), array.shape), TF_DataType.TF_INT32 => Render(array.ToArray(), array.shape), TF_DataType.TF_INT64 => Render(array.ToArray(), array.shape), + TF_DataType.TF_UINT64 => Render(array.ToArray(), array.shape), TF_DataType.TF_FLOAT => Render(array.ToArray(), array.shape), TF_DataType.TF_DOUBLE => Render(array.ToArray(), array.shape), _ => Render(array.ToArray(), array.shape) diff --git a/src/TensorFlowNET.Core/Operations/Initializers/Orthogonal.cs b/src/TensorFlowNET.Core/Operations/Initializers/Orthogonal.cs index 90f3f93c3..045b02c5a 100644 --- a/src/TensorFlowNET.Core/Operations/Initializers/Orthogonal.cs +++ b/src/TensorFlowNET.Core/Operations/Initializers/Orthogonal.cs @@ -1,32 +1,62 @@ -using System; +/***************************************************************************** + Copyright 2023 Haiping Chen. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +using System; using System.Linq; -using static Tensorflow.TensorShapeProto.Types; +using static Tensorflow.Binding; -namespace Tensorflow.Operations.Initializers +namespace Tensorflow.Operations.Initializers; + +public class Orthogonal : IInitializer { - public class Orthogonal : IInitializer + float _gain = 0f; + int? _seed; + + public Orthogonal(float gain = 1.0f, int? seed = null) { - float _gain = 0f; + _gain = gain; + _seed = seed; + } - public Orthogonal(float gain = 1.0f, int? seed = null) - { + public Tensor Apply(InitializerArgs args) + { + return _generate_init_val(args.Shape, args.DType == TF_DataType.DtInvalid ? TF_DataType.TF_FLOAT : args.DType); + } - } + private Tensor _generate_init_val(Shape shape, TF_DataType dtype) + { + var num_rows = 1L; + foreach (var dim in shape.dims.Take(shape.ndim - 1)) + num_rows *= dim; + var num_cols = shape.dims.Last(); + var flat_shape = (Math.Max(num_cols, num_rows), Math.Min(num_cols, num_rows)); - public Tensor Apply(InitializerArgs args) - { - return _generate_init_val(args.Shape, args.DType); - } + var a = tf.random.stateless_normal(flat_shape, dtype: dtype); + // Compute the qr factorization + var (q, r) = tf.linalg.qr(a, full_matrices: false); + // Make Q uniform + var d = tf.linalg.tensor_diag_part(r); + q *= tf.sign(d); - private Tensor _generate_init_val(Shape shape, TF_DataType dtype) + if (num_rows < num_cols) { - var num_rows = 1L; - foreach (var dim in shape.dims.Take(shape.ndim - 1)) - num_rows *= dim; - var num_cols = shape.dims.Last(); - var flat_shape = (Math.Max(num_cols, num_rows), Math.Min(num_cols, num_rows)); - + // q = tf.linalg.matrix_transpose(q); throw new NotImplementedException(""); } + + return _gain * tf.reshape(q, shape); } } diff --git a/src/TensorFlowNET.Core/Operations/gen_array_ops.cs b/src/TensorFlowNET.Core/Operations/gen_array_ops.cs index dd1604f61..794c32673 100644 --- a/src/TensorFlowNET.Core/Operations/gen_array_ops.cs +++ b/src/TensorFlowNET.Core/Operations/gen_array_ops.cs @@ -113,6 +113,9 @@ public static Tensor[] concat_offset(Tensor concat_dim, Tensor[] shape, string n public static Tensor diag(Tensor diagonal, string name = null) => tf.Context.ExecuteOp("Diag", name, new ExecuteOpArgs(diagonal)); + public static Tensor diag_part(Tensor diagonal, string name = null) + => tf.Context.ExecuteOp("DiagPart", name, new ExecuteOpArgs(diagonal)); + public static Tensor expand_dims(Tensor input, int axis, string name = null) => tf.Context.ExecuteOp("ExpandDims", name, new ExecuteOpArgs(input, axis) .SetAttributes(new { dim = axis })); diff --git a/src/TensorFlowNET.Core/Operations/gen_random_ops.cs b/src/TensorFlowNET.Core/Operations/gen_random_ops.cs index 0edea3aac..a6cc47182 100644 --- a/src/TensorFlowNET.Core/Operations/gen_random_ops.cs +++ b/src/TensorFlowNET.Core/Operations/gen_random_ops.cs @@ -13,7 +13,10 @@ You may obtain a copy of the License at See the License for the specific language governing permissions and limitations under the License. ******************************************************************************/ +using static Tensorflow.ApiDef.Types; +using System.Reflection; using static Tensorflow.Binding; +using System.Xml.Linq; namespace Tensorflow { @@ -85,6 +88,15 @@ public static Tensor truncated_normal(Tensor shape, TF_DataType dtype, int? seed int? seed2 = 0, string name = null) => tf.Context.ExecuteOp("TruncatedNormal", name, new ExecuteOpArgs(shape) .SetAttributes(new { dtype, seed = seed ?? 0, seed2 = seed2 ?? 0 })); + public static Tensor stateless_random_normal_v2(Tensor shape, Tensor key, Tensor counter, + int alg, TF_DataType dtype, string name = null) + => tf.Context.ExecuteOp("StatelessRandomNormalV2", name, + new ExecuteOpArgs(shape, key, counter, alg) + .SetAttributes(new { dtype })); + + public static Tensors stateless_random_get_key_counter(int[] seed, string name = null) + => tf.Context.ExecuteOp("StatelessRandomGetKeyCounter", name, + new ExecuteOpArgs(seed)); public static Tensor multinomial(Tensor logits, int num_samples, int? seed = 0, int? seed2 = 0, TF_DataType output_dtype = TF_DataType.TF_INT64, string name = null) diff --git a/src/TensorFlowNET.Core/Operations/linalg_ops.cs b/src/TensorFlowNET.Core/Operations/linalg_ops.cs index 024ea14d9..42da1a279 100644 --- a/src/TensorFlowNET.Core/Operations/linalg_ops.cs +++ b/src/TensorFlowNET.Core/Operations/linalg_ops.cs @@ -129,5 +129,12 @@ public Tensor matrix_triangular_solve(Tensor matrix, Tensor rhs, bool lower = tr lower, adjoint })); + + public Tensors qr(Tensor input, bool full_matrices = false, string name = null) + => tf.Context.ExecuteOp("Qr", name, + new ExecuteOpArgs(input).SetAttributes(new + { + full_matrices + })); } } diff --git a/src/TensorFlowNET.Core/Operations/stateless_random_ops.cs b/src/TensorFlowNET.Core/Operations/stateless_random_ops.cs new file mode 100644 index 000000000..e9718770c --- /dev/null +++ b/src/TensorFlowNET.Core/Operations/stateless_random_ops.cs @@ -0,0 +1,62 @@ +/***************************************************************************** + Copyright 2023 Haiping Chen. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +using static Tensorflow.ApiDef.Types; +using System.Reflection; +using static Tensorflow.Binding; +using System; + +namespace Tensorflow; + +public class stateless_random_ops +{ + public static Tensor stateless_random_normal(Shape shape, + float mean = 0.0f, + float stddev = 1.0f, + TF_DataType dtype = TF_DataType.TF_FLOAT, + int[]? seed = null, + string name = null) + { + return tf_with(ops.name_scope(name, "stateless_random_normal", new { shape, seed, mean, stddev }), scope => + { + name = scope; + var shape_tensor = _ShapeTensor(shape); + var mean_tensor = ops.convert_to_tensor(mean, dtype: dtype, name: "mean"); + var stddev_tensor = ops.convert_to_tensor(stddev, dtype: dtype, name: "stddev"); + + if (seed == null) + { + seed = new[] { new Random().Next(), 0 }; + } + var (key, counter) = _get_key_counter(seed, 3); + var rnd = gen_random_ops.stateless_random_normal_v2(shape: shape_tensor, key: key, counter: counter, dtype: dtype, alg: 3); + var value = math_ops.add(rnd * stddev, mean_tensor, name: name); + // tensor_util.maybe_set_static_shape(value, shape) + return value; + }); + } + + private static Tensor _ShapeTensor(int[] shape) + { + return ops.convert_to_tensor(shape, name: "shape"); + } + + private static (Tensor, Tensor) _get_key_counter(int[] seed, int alg) + { + var results = gen_random_ops.stateless_random_get_key_counter(seed); + return (results[0], results[1]); + } +} diff --git a/src/TensorFlowNET.Core/tensorflow.cs b/src/TensorFlowNET.Core/tensorflow.cs index 35762be12..6e655a196 100644 --- a/src/TensorFlowNET.Core/tensorflow.cs +++ b/src/TensorFlowNET.Core/tensorflow.cs @@ -67,7 +67,10 @@ public tensorflow() public void UseKeras() where T : IKerasApi, new() { - keras = new T(); + if (keras == null) + { + keras = new T(); + } } public string VERSION => c_api.StringPiece(c_api.TF_Version()); diff --git a/src/TensorFlowNET.Keras/Initializers.cs b/src/TensorFlowNET.Keras/InitializersApi.cs similarity index 64% rename from src/TensorFlowNET.Keras/Initializers.cs rename to src/TensorFlowNET.Keras/InitializersApi.cs index b432cc97c..d37ccd99b 100644 --- a/src/TensorFlowNET.Keras/Initializers.cs +++ b/src/TensorFlowNET.Keras/InitializersApi.cs @@ -16,18 +16,20 @@ limitations under the License. using Tensorflow.Operations.Initializers; -namespace Tensorflow.Keras +namespace Tensorflow.Keras; + +public partial class InitializersApi : IInitializersApi { - public class Initializers + /// + /// He normal initializer. + /// + /// + /// + public IInitializer he_normal(int? seed = null) { - /// - /// He normal initializer. - /// - /// - /// - public IInitializer he_normal(int? seed = null) - { - return new VarianceScaling(factor: 2.0f, mode: "fan_in", seed: seed); - } + return new VarianceScaling(factor: 2.0f, mode: "fan_in", seed: seed); } + + public IInitializer Orthogonal(float gain = 1.0f, int? seed = null) + => new Orthogonal(gain: gain, seed: seed); } diff --git a/src/TensorFlowNET.Keras/KerasInterface.cs b/src/TensorFlowNET.Keras/KerasInterface.cs index 5bf9f97f3..8dde1ab41 100644 --- a/src/TensorFlowNET.Keras/KerasInterface.cs +++ b/src/TensorFlowNET.Keras/KerasInterface.cs @@ -18,7 +18,7 @@ namespace Tensorflow.Keras public class KerasInterface : IKerasApi { public KerasDataset datasets { get; } = new KerasDataset(); - public Initializers initializers { get; } = new Initializers(); + public IInitializersApi initializers { get; } = new InitializersApi(); public Regularizers regularizers { get; } = new Regularizers(); public ILayersApi layers { get; } = new LayersApi(); public LossesApi losses { get; } = new LossesApi(); diff --git a/test/TensorFlowNET.Keras.UnitTest/EagerModeTestBase.cs b/test/TensorFlowNET.Keras.UnitTest/EagerModeTestBase.cs index 566ade306..04ed3df4d 100644 --- a/test/TensorFlowNET.Keras.UnitTest/EagerModeTestBase.cs +++ b/test/TensorFlowNET.Keras.UnitTest/EagerModeTestBase.cs @@ -1,5 +1,6 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using System; +using Tensorflow.Keras; using static Tensorflow.Binding; namespace TensorFlowNET.Keras.UnitTest @@ -9,6 +10,8 @@ public class EagerModeTestBase [TestInitialize] public void TestInit() { + tf.UseKeras(); + if (!tf.executing_eagerly()) tf.enable_eager_execution(); tf.Context.ensure_initialized(); diff --git a/test/TensorFlowNET.Keras.UnitTest/InitializerTest.cs b/test/TensorFlowNET.Keras.UnitTest/InitializerTest.cs new file mode 100644 index 000000000..c811b5643 --- /dev/null +++ b/test/TensorFlowNET.Keras.UnitTest/InitializerTest.cs @@ -0,0 +1,20 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using TensorFlowNET.Keras.UnitTest; +using static Tensorflow.Binding; + +namespace Tensorflow.Keras.UnitTest; + +[TestClass] +public class InitializerTest : EagerModeTestBase +{ + [TestMethod] + public void Orthogonal() + { + var initializer = tf.keras.initializers.Orthogonal(); + var values = initializer.Apply(new InitializerArgs((2, 2))); + } +} From 56a64dae2e19946bc758b81d2901277d589c8982 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sat, 21 Jan 2023 12:52:32 -0600 Subject: [PATCH 14/52] np.sort --- .../NumPy/NumPy.Sorting.Searching.Counting.cs | 22 ++++++++++++++-- src/TensorFlowNET.Core/Operations/sort_ops.cs | 25 +++++++++++++++++++ .../NumPy/Array.Sorting.Test.cs | 12 ++++++++- 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/src/TensorFlowNET.Core/NumPy/NumPy.Sorting.Searching.Counting.cs b/src/TensorFlowNET.Core/NumPy/NumPy.Sorting.Searching.Counting.cs index 61feb5e78..5182d5726 100644 --- a/src/TensorFlowNET.Core/NumPy/NumPy.Sorting.Searching.Counting.cs +++ b/src/TensorFlowNET.Core/NumPy/NumPy.Sorting.Searching.Counting.cs @@ -1,6 +1,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Globalization; using System.Numerics; using System.Text; @@ -9,11 +10,11 @@ namespace Tensorflow.NumPy public partial class np { [AutoNumPy] - public static NDArray argmax(NDArray a, Axis axis = null) + public static NDArray argmax(NDArray a, Axis? axis = null) => new NDArray(math_ops.argmax(a, axis ?? 0)); [AutoNumPy] - public static NDArray argsort(NDArray a, Axis axis = null) + public static NDArray argsort(NDArray a, Axis? axis = null) => new NDArray(sort_ops.argsort(a, axis: axis ?? -1)); [AutoNumPy] @@ -25,5 +26,22 @@ public static (NDArray, NDArray) unique(NDArray a) [AutoNumPy] public static void shuffle(NDArray x) => np.random.shuffle(x); + + /// + /// Sorts a ndarray + /// + /// + /// + /// The axis along which to sort. The default is -1, which sorts the last axis. + /// + /// + /// The direction in which to sort the values (`'ASCENDING'` or `'DESCENDING'`) + /// + /// + /// A `NDArray` with the same dtype and shape as `values`, with the elements sorted along the given `axis`. + /// + [AutoNumPy] + public static NDArray sort(NDArray values, Axis? axis = null, string direction = "ASCENDING") + => new NDArray(sort_ops.sort(values, axis: axis ?? -1, direction: direction)); } } diff --git a/src/TensorFlowNET.Core/Operations/sort_ops.cs b/src/TensorFlowNET.Core/Operations/sort_ops.cs index 1dcaf1f84..34b903230 100644 --- a/src/TensorFlowNET.Core/Operations/sort_ops.cs +++ b/src/TensorFlowNET.Core/Operations/sort_ops.cs @@ -47,6 +47,31 @@ public static Tensor argsort(Tensor values, Axis axis = null, string direction = return indices; } + public static Tensor sort(Tensor values, Axis axis, string direction = "ASCENDING", string? name = null) + { + var k = array_ops.shape(values)[axis]; + values = -values; + var static_rank = values.shape.ndim; + var top_k_input = values; + if (axis == -1 || axis + 1 == values.shape.ndim) + { + } + else + { + if (axis == 0 && static_rank == 2) + top_k_input = array_ops.transpose(values, new[] { 1, 0 }); + else + throw new NotImplementedException(""); + } + + (values, _) = tf.Context.ExecuteOp("TopKV2", name, + new ExecuteOpArgs(top_k_input, k).SetAttributes(new + { + sorted = true + })); + return -values; + } + public Tensor matrix_inverse(Tensor input, bool adjoint = false, string name = null) => tf.Context.ExecuteOp("MatrixInverse", name, new ExecuteOpArgs(input).SetAttributes(new diff --git a/test/TensorFlowNET.UnitTest/NumPy/Array.Sorting.Test.cs b/test/TensorFlowNET.UnitTest/NumPy/Array.Sorting.Test.cs index 2a617d409..13a5d9739 100644 --- a/test/TensorFlowNET.UnitTest/NumPy/Array.Sorting.Test.cs +++ b/test/TensorFlowNET.UnitTest/NumPy/Array.Sorting.Test.cs @@ -5,7 +5,6 @@ using System.Text; using Tensorflow; using Tensorflow.NumPy; -using static Tensorflow.Binding; namespace TensorFlowNET.UnitTest.NumPy { @@ -30,5 +29,16 @@ public void argsort() Assert.AreEqual(ind[0], new[] { 0, 1 }); Assert.AreEqual(ind[1], new[] { 1, 0 }); } + + /// + /// https://numpy.org/doc/stable/reference/generated/numpy.sort.html + /// + [TestMethod] + public void sort() + { + var x = np.array(new int[] { 3, 1, 2 }); + var sorted = np.sort(x); + Assert.IsTrue(sorted.ToArray() is [1, 2, 3]); + } } } From 4f88109ae31229f4137045e959400a8342874036 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sat, 28 Jan 2023 18:45:39 -0600 Subject: [PATCH 15/52] fix _self_tracked_trackables. --- src/TensorFlowNET.Core/Keras/IInitializersApi.cs | 2 ++ src/TensorFlowNET.Core/Tensorflow.Binding.csproj | 12 ++++++------ src/TensorFlowNET.Keras/Engine/Functional.cs | 7 +------ src/TensorFlowNET.Keras/Engine/Sequential.cs | 4 +++- src/TensorFlowNET.Keras/InitializersApi.cs | 2 +- src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs | 5 +++++ src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs | 7 ++++++- src/TensorFlowNET.Keras/Tensorflow.Keras.csproj | 8 ++++---- 8 files changed, 28 insertions(+), 19 deletions(-) diff --git a/src/TensorFlowNET.Core/Keras/IInitializersApi.cs b/src/TensorFlowNET.Core/Keras/IInitializersApi.cs index ff92040eb..3ad5e87b8 100644 --- a/src/TensorFlowNET.Core/Keras/IInitializersApi.cs +++ b/src/TensorFlowNET.Core/Keras/IInitializersApi.cs @@ -7,5 +7,7 @@ namespace Tensorflow.Keras public interface IInitializersApi { IInitializer Orthogonal(float gain = 1.0f, int? seed = null); + + IInitializer HeNormal(int? seed = null); } } diff --git a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj index 0ebe61d0d..4c42cb8c3 100644 --- a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj +++ b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj @@ -1,11 +1,11 @@  - netstandard2.0 + netstandard2.0;net6.0 Tensorflow.Binding Tensorflow - 2.2.0 - 0.100.0 + 2.10.0 + 0.100.1 10.0 enable Haiping Chen, Meinrad Recheis, Eli Belash @@ -20,7 +20,7 @@ Google's TensorFlow full binding in .NET Standard. Building, training and infering deep learning models. https://tensorflownet.readthedocs.io - 0.100.0.0 + 0.100.1.0 tf.net 0.100.x and above are based on tensorflow native 2.10.0 @@ -38,7 +38,7 @@ https://tensorflownet.readthedocs.io tf.net 0.7x.x aligns with TensorFlow v2.7.x native library. tf.net 0.10x.x aligns with TensorFlow v2.10.x native library. - 0.100.0.0 + 0.100.1.0 LICENSE true true @@ -108,7 +108,7 @@ https://tensorflownet.readthedocs.io - + diff --git a/src/TensorFlowNET.Keras/Engine/Functional.cs b/src/TensorFlowNET.Keras/Engine/Functional.cs index d10ed214a..09a31b948 100644 --- a/src/TensorFlowNET.Keras/Engine/Functional.cs +++ b/src/TensorFlowNET.Keras/Engine/Functional.cs @@ -65,12 +65,7 @@ protected void _init_graph_network(Tensors inputs, Tensors outputs) } // Keep track of the network's nodes and layers. - (NetworkNodes, NodesByDepth, var layers, _) = MapGraphNetwork(inputs, outputs); - - if (!_self_tracked_trackables.Any()) - { - _self_tracked_trackables = layers; - } + (NetworkNodes, NodesByDepth, _self_tracked_trackables, _) = MapGraphNetwork(inputs, outputs); // Build self.input_names and self.output_names. _set_output_names(); diff --git a/src/TensorFlowNET.Keras/Engine/Sequential.cs b/src/TensorFlowNET.Keras/Engine/Sequential.cs index b4d1ecfef..4d87659bd 100644 --- a/src/TensorFlowNET.Keras/Engine/Sequential.cs +++ b/src/TensorFlowNET.Keras/Engine/Sequential.cs @@ -110,6 +110,8 @@ public void add(ILayer layer) } else if (outputs != null) { + // If the model is being built continuously on top of an input layer: + // refresh its output. outputs = layer.Apply(outputs); built = true; } @@ -155,7 +157,7 @@ void _build_graph_network_for_inferred_shape(Shape input_shape, TF_DataType inpu Tensors layer_output = null; Tensors outputs = null; List created_nodes = new List(); - foreach (var layer in _self_tracked_trackables) + foreach (var layer in args.Layers) { clear_previously_created_nodes(layer, _created_nodes); layer_output = layer.Apply(layer_input); diff --git a/src/TensorFlowNET.Keras/InitializersApi.cs b/src/TensorFlowNET.Keras/InitializersApi.cs index d37ccd99b..6bade1720 100644 --- a/src/TensorFlowNET.Keras/InitializersApi.cs +++ b/src/TensorFlowNET.Keras/InitializersApi.cs @@ -25,7 +25,7 @@ public partial class InitializersApi : IInitializersApi /// /// /// - public IInitializer he_normal(int? seed = null) + public IInitializer HeNormal(int? seed = null) { return new VarianceScaling(factor: 2.0f, mode: "fan_in", seed: seed); } diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs index f894f41ff..877c35994 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs @@ -45,6 +45,11 @@ public override void build(Shape input_shape) } } + protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + { + return base.Call(inputs, state, training); + } + private static RNNArgs PreConstruct(RNNArgs args) { if (args.Kwargs == null) diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs index 8d696d160..9e5af450b 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNNCell.cs @@ -33,11 +33,16 @@ public override void build(Shape input_shape) if (args.UseBias) { bias = add_weight("bias", (args.Units), - initializer: args.RecurrentInitializer + initializer: args.BiasInitializer ); } built = true; } + + protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + { + return base.Call(inputs, state, training); + } } } diff --git a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj index 647601a77..d45c7de2e 100644 --- a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj +++ b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj @@ -1,13 +1,13 @@  - netstandard2.0 + netstandard2.0;net6.0 Tensorflow.Keras 10.0 enable Tensorflow.Keras AnyCPU;x64 - 0.10.0 + 0.10.1 Haiping Chen Keras for .NET Apache 2.0, Haiping Chen 2021 @@ -37,8 +37,8 @@ Keras is an API designed for human beings, not machines. Keras follows best prac Git true Open.snk - 0.10.0.0 - 0.10.0.0 + 0.10.1.0 + 0.10.1.0 LICENSE Debug;Release;GPU From f48ba40263ed18109b29a75cdcf31abb4be08760 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sun, 29 Jan 2023 10:40:07 -0600 Subject: [PATCH 16/52] Fix MaxPooling1D #969 --- .../Operations/NnOps/MaxPoolFunction.cs | 6 ++--- .../Tensorflow.Binding.csproj | 6 ++--- .../Layers/Pooling/Pooling1D.cs | 22 ++++++++++++------- .../Layers/Pooling/Pooling2D.cs | 2 +- .../Tensorflow.Keras.csproj | 8 +++---- .../Layers/PoolingTest.cs | 5 +++-- 6 files changed, 28 insertions(+), 21 deletions(-) diff --git a/src/TensorFlowNET.Core/Operations/NnOps/MaxPoolFunction.cs b/src/TensorFlowNET.Core/Operations/NnOps/MaxPoolFunction.cs index 92bd95a57..149d2e889 100644 --- a/src/TensorFlowNET.Core/Operations/NnOps/MaxPoolFunction.cs +++ b/src/TensorFlowNET.Core/Operations/NnOps/MaxPoolFunction.cs @@ -14,6 +14,7 @@ You may obtain a copy of the License at limitations under the License. ******************************************************************************/ +using System.Linq; using static Tensorflow.Binding; namespace Tensorflow.Operations @@ -24,7 +25,7 @@ namespace Tensorflow.Operations public class MaxPoolFunction : IPoolFunction { public Tensor Apply(Tensor value, - int[] ksize, + int[] pool_size, int[] strides, string padding, string data_format = "NHWC", @@ -33,10 +34,9 @@ public Tensor Apply(Tensor value, return tf_with(ops.name_scope(name, "MaxPool", value), scope => { name = scope; - value = ops.convert_to_tensor(value, name: "input"); return gen_nn_ops.max_pool( value, - ksize: ksize, + ksize: pool_size, strides: strides, padding: padding, data_format: data_format, diff --git a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj index 4c42cb8c3..a7db6eee1 100644 --- a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj +++ b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj @@ -5,7 +5,7 @@ Tensorflow.Binding Tensorflow 2.10.0 - 0.100.1 + 0.100.2 10.0 enable Haiping Chen, Meinrad Recheis, Eli Belash @@ -20,7 +20,7 @@ Google's TensorFlow full binding in .NET Standard. Building, training and infering deep learning models. https://tensorflownet.readthedocs.io - 0.100.1.0 + 0.100.2.0 tf.net 0.100.x and above are based on tensorflow native 2.10.0 @@ -38,7 +38,7 @@ https://tensorflownet.readthedocs.io tf.net 0.7x.x aligns with TensorFlow v2.7.x native library. tf.net 0.10x.x aligns with TensorFlow v2.10.x native library. - 0.100.1.0 + 0.100.2.0 LICENSE true true diff --git a/src/TensorFlowNET.Keras/Layers/Pooling/Pooling1D.cs b/src/TensorFlowNET.Keras/Layers/Pooling/Pooling1D.cs index 80b36c86d..a2f4c51b6 100644 --- a/src/TensorFlowNET.Keras/Layers/Pooling/Pooling1D.cs +++ b/src/TensorFlowNET.Keras/Layers/Pooling/Pooling1D.cs @@ -14,9 +14,11 @@ You may obtain a copy of the License at limitations under the License. ******************************************************************************/ +using System.Linq; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Utils; +using static Tensorflow.Binding; namespace Tensorflow.Keras.Layers { @@ -36,17 +38,21 @@ public Pooling1D(Pooling1DArgs args) protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) { - int[] pool_shape; - int[] strides; + int pad_axis = args.DataFormat == "channels_first" ? 2 : 3; + inputs = tf.expand_dims(inputs, pad_axis); + int[] pool_shape = new int[] { args.PoolSize, 1 }; + int[] strides = new int[] { args.Strides, 1 }; + var ndim = inputs[0].ndim; + if (args.DataFormat == "channels_last") { - pool_shape = new int[] { 1, args.PoolSize, 1 }; - strides = new int[] { 1, args.Strides, 1 }; + pool_shape = new int[] { 1 }.Concat(pool_shape).Concat(new int[] { 1 }).ToArray(); + strides = new int[] { 1 }.Concat(strides).Concat(new int[] { 1 }).ToArray(); } else { - pool_shape = new int[] { 1, 1, args.PoolSize }; - strides = new int[] { 1, 1, args.Strides }; + pool_shape = new int[] { 1, 1 }.Concat(pool_shape).ToArray(); + strides = new int[] { 1, 1 }.Concat(strides).ToArray(); } var outputs = args.PoolFunction.Apply( @@ -54,9 +60,9 @@ protected override Tensors Call(Tensors inputs, Tensor state = null, bool? train ksize: pool_shape, strides: strides, padding: args.Padding.ToUpper(), - data_format: conv_utils.convert_data_format(args.DataFormat, 3)); + data_format: conv_utils.convert_data_format(args.DataFormat, ndim)); - return outputs; + return tf.squeeze(outputs, pad_axis); } } } diff --git a/src/TensorFlowNET.Keras/Layers/Pooling/Pooling2D.cs b/src/TensorFlowNET.Keras/Layers/Pooling/Pooling2D.cs index e65bf0388..270322559 100644 --- a/src/TensorFlowNET.Keras/Layers/Pooling/Pooling2D.cs +++ b/src/TensorFlowNET.Keras/Layers/Pooling/Pooling2D.cs @@ -42,7 +42,7 @@ protected override Tensors Call(Tensors inputs, Tensor state = null, bool? train int[] strides; if (args.DataFormat == "channels_last") { - pool_shape = new int[] { 1, (int)args.PoolSize.dims[0], (int)args.PoolSize.dims[1], 1 }; + pool_shape = new int[] { 1, (int)args.PoolSize.dims[0], (int)args.PoolSize.dims[1], 1 }; strides = new int[] { 1, (int)args.Strides.dims[0], (int)args.Strides.dims[1], 1 }; } else diff --git a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj index d45c7de2e..f7d186355 100644 --- a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj +++ b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj @@ -7,7 +7,7 @@ enable Tensorflow.Keras AnyCPU;x64 - 0.10.1 + 0.10.2 Haiping Chen Keras for .NET Apache 2.0, Haiping Chen 2021 @@ -37,8 +37,8 @@ Keras is an API designed for human beings, not machines. Keras follows best prac Git true Open.snk - 0.10.1.0 - 0.10.1.0 + 0.10.2.0 + 0.10.2.0 LICENSE Debug;Release;GPU @@ -70,7 +70,7 @@ Keras is an API designed for human beings, not machines. Keras follows best prac - + diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/PoolingTest.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/PoolingTest.cs index 8af408555..0eab0a986 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/PoolingTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/PoolingTest.cs @@ -4,6 +4,7 @@ using Tensorflow; using static Tensorflow.Binding; using static Tensorflow.KerasApi; +using Microsoft.VisualBasic; namespace TensorFlowNET.Keras.UnitTest { @@ -226,7 +227,7 @@ public void GlobalMax2DPoolingChannelsFirst() Assert.AreEqual(expected, y[0].numpy()); } - [TestMethod, Ignore("There's an error generated from TF complaining about the shape of the pool. Needs further investigation.")] + [TestMethod] public void Max1DPoolingChannelsLast() { var x = input_array_1D; @@ -239,7 +240,7 @@ public void Max1DPoolingChannelsLast() var expected = np.array(new float[,,] { - {{2.0f, 2.0f, 3.0f, 3.0f, 3.0f}, + {{1.0f, 2.0f, 3.0f, 3.0f, 3.0f}, { 1.0f, 2.0f, 3.0f, 3.0f, 3.0f}}, {{4.0f, 5.0f, 6.0f, 3.0f, 3.0f}, From c5cdf2c540922ac3abf38e53d1b00e4241802e35 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Mon, 30 Jan 2023 20:08:40 -0600 Subject: [PATCH 17/52] Fixed model.fit return results. #927 --- .../NumPy/Numpy.Manipulation.cs | 4 + .../Callbacks/CallbackList.cs | 43 ++++++++++ .../Callbacks/CallbackParams.cs | 15 ++++ src/TensorFlowNET.Keras/Callbacks/History.cs | 52 ++++++++++++ .../Callbacks/ICallback.cs | 15 ++++ .../Callbacks/ProgbarLogger.cs | 81 +++++++++++++++++++ .../Engine/Model.Evaluate.cs | 18 ++--- src/TensorFlowNET.Keras/Engine/Model.Fit.cs | 79 ++++++++---------- src/TensorFlowNET.Keras/Engine/Model.Train.cs | 11 ++- src/TensorFlowNET.Keras/Engine/Model.cs | 1 - .../Layers/LayersTest.cs | 2 +- 11 files changed, 258 insertions(+), 63 deletions(-) create mode 100644 src/TensorFlowNET.Keras/Callbacks/CallbackList.cs create mode 100644 src/TensorFlowNET.Keras/Callbacks/CallbackParams.cs create mode 100644 src/TensorFlowNET.Keras/Callbacks/History.cs create mode 100644 src/TensorFlowNET.Keras/Callbacks/ICallback.cs create mode 100644 src/TensorFlowNET.Keras/Callbacks/ProgbarLogger.cs diff --git a/src/TensorFlowNET.Core/NumPy/Numpy.Manipulation.cs b/src/TensorFlowNET.Core/NumPy/Numpy.Manipulation.cs index 091509fda..940856056 100644 --- a/src/TensorFlowNET.Core/NumPy/Numpy.Manipulation.cs +++ b/src/TensorFlowNET.Core/NumPy/Numpy.Manipulation.cs @@ -8,6 +8,10 @@ namespace Tensorflow.NumPy { public partial class np { + [AutoNumPy] + public static NDArray concatenate((NDArray, NDArray) tuple, int axis = 0) + => new NDArray(array_ops.concat(new[] { tuple.Item1, tuple.Item2 }, axis)); + [AutoNumPy] public static NDArray concatenate(NDArray[] arrays, int axis = 0) => new NDArray(array_ops.concat(arrays, axis)); diff --git a/src/TensorFlowNET.Keras/Callbacks/CallbackList.cs b/src/TensorFlowNET.Keras/Callbacks/CallbackList.cs new file mode 100644 index 000000000..bb3ed6edc --- /dev/null +++ b/src/TensorFlowNET.Keras/Callbacks/CallbackList.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.Callbacks +{ + public class CallbackList + { + List callbacks = new List(); + public History History => callbacks[0] as History; + + public CallbackList(CallbackParams parameters) + { + callbacks.Add(new History(parameters)); + callbacks.Add(new ProgbarLogger(parameters)); + } + + public void on_train_begin() + { + callbacks.ForEach(x => x.on_train_begin()); + } + + public void on_epoch_begin(int epoch) + { + callbacks.ForEach(x => x.on_epoch_begin(epoch)); + } + + public void on_train_batch_begin(long step) + { + callbacks.ForEach(x => x.on_train_batch_begin(step)); + } + + public void on_train_batch_end(long end_step, Dictionary logs) + { + callbacks.ForEach(x => x.on_train_batch_end(end_step, logs)); + } + + public void on_epoch_end(int epoch, Dictionary epoch_logs) + { + callbacks.ForEach(x => x.on_epoch_end(epoch, epoch_logs)); + } + } +} diff --git a/src/TensorFlowNET.Keras/Callbacks/CallbackParams.cs b/src/TensorFlowNET.Keras/Callbacks/CallbackParams.cs new file mode 100644 index 000000000..fe859c8a2 --- /dev/null +++ b/src/TensorFlowNET.Keras/Callbacks/CallbackParams.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Keras.Engine; + +namespace Tensorflow.Keras.Callbacks +{ + public class CallbackParams + { + public IModel Model { get; set; } + public int Verbose { get; set; } + public int Epochs { get; set; } + public long Steps { get; set; } + } +} diff --git a/src/TensorFlowNET.Keras/Callbacks/History.cs b/src/TensorFlowNET.Keras/Callbacks/History.cs new file mode 100644 index 000000000..02588b5e7 --- /dev/null +++ b/src/TensorFlowNET.Keras/Callbacks/History.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.Callbacks +{ + public class History : ICallback + { + List epochs; + CallbackParams _parameters; + public Dictionary> history { get; set; } + + public History(CallbackParams parameters) + { + _parameters = parameters; + } + + public void on_train_begin() + { + epochs = new List(); + history = new Dictionary>(); + } + + public void on_epoch_begin(int epoch) + { + + } + + public void on_train_batch_begin(long step) + { + + } + + public void on_train_batch_end(long end_step, Dictionary logs) + { + } + + public void on_epoch_end(int epoch, Dictionary epoch_logs) + { + epochs.Add(epoch); + + foreach (var log in epoch_logs) + { + if (!history.ContainsKey(log.Key)) + { + history[log.Key] = new List(); + } + history[log.Key].Add((float)log.Value); + } + } + } +} diff --git a/src/TensorFlowNET.Keras/Callbacks/ICallback.cs b/src/TensorFlowNET.Keras/Callbacks/ICallback.cs new file mode 100644 index 000000000..34763c557 --- /dev/null +++ b/src/TensorFlowNET.Keras/Callbacks/ICallback.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.Callbacks +{ + public interface ICallback + { + void on_train_begin(); + void on_epoch_begin(int epoch); + void on_train_batch_begin(long step); + void on_train_batch_end(long end_step, Dictionary logs); + void on_epoch_end(int epoch, Dictionary epoch_logs); + } +} diff --git a/src/TensorFlowNET.Keras/Callbacks/ProgbarLogger.cs b/src/TensorFlowNET.Keras/Callbacks/ProgbarLogger.cs new file mode 100644 index 000000000..17e041014 --- /dev/null +++ b/src/TensorFlowNET.Keras/Callbacks/ProgbarLogger.cs @@ -0,0 +1,81 @@ +using PureHDF; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; + +namespace Tensorflow.Keras.Callbacks +{ + public class ProgbarLogger : ICallback + { + bool _called_in_fit = false; + int seen = 0; + CallbackParams _parameters; + Stopwatch _sw; + + public ProgbarLogger(CallbackParams parameters) + { + _parameters = parameters; + } + + public void on_train_begin() + { + _called_in_fit = true; + _sw = new Stopwatch(); + } + + public void on_epoch_begin(int epoch) + { + _reset_progbar(); + _maybe_init_progbar(); + Binding.tf_output_redirect.WriteLine($"Epoch: {epoch + 1:D3}/{_parameters.Epochs:D3}"); + } + + public void on_train_batch_begin(long step) + { + _sw.Restart(); + } + + public void on_train_batch_end(long end_step, Dictionary logs) + { + _sw.Stop(); + var elapse = _sw.ElapsedMilliseconds; + var results = string.Join(" - ", logs.Select(x => $"{x.Key}: {(float)x.Value:F6}")); + + var progress = ""; + var length = 30.0 / _parameters.Steps; + for (int i = 0; i < Math.Floor(end_step * length - 1); i++) + progress += "="; + if (progress.Length < 28) + progress += ">"; + else + progress += "="; + + var remaining = ""; + for (int i = 1; i < 30 - progress.Length; i++) + remaining += "."; + + Binding.tf_output_redirect.Write($"{end_step + 1:D4}/{_parameters.Steps:D4} [{progress}{remaining}] - {elapse}ms/step - {results}"); + if (!Console.IsOutputRedirected) + { + Console.CursorLeft = 0; + } + } + + public void on_epoch_end(int epoch, Dictionary epoch_logs) + { + Console.WriteLine(); + } + + void _reset_progbar() + { + seen = 0; + } + + void _maybe_init_progbar() + { + + } + } +} diff --git a/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs b/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs index 98e02ed36..c9d398339 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Evaluate.cs @@ -31,7 +31,7 @@ public void evaluate(NDArray x, NDArray y, bool use_multiprocessing = false, bool return_dict = false) { - data_handler = new DataHandler(new DataHandlerArgs + var data_handler = new DataHandler(new DataHandlerArgs { X = x, Y = y, @@ -46,7 +46,6 @@ public void evaluate(NDArray x, NDArray y, StepsPerExecution = _steps_per_execution }); - Binding.tf_output_redirect.WriteLine($"Testing..."); foreach (var (epoch, iterator) in data_handler.enumerate_epochs()) { reset_metrics(); @@ -56,22 +55,20 @@ public void evaluate(NDArray x, NDArray y, foreach (var step in data_handler.steps()) { // callbacks.on_train_batch_begin(step) - results = test_function(iterator); + results = test_function(data_handler, iterator); } - Binding.tf_output_redirect.WriteLine($"iterator: {epoch + 1}, " + string.Join(", ", results.Select(x => $"{x.Item1}: {(float)x.Item2}"))); } } public KeyValuePair[] evaluate(IDatasetV2 x) { - data_handler = new DataHandler(new DataHandlerArgs + var data_handler = new DataHandler(new DataHandlerArgs { Dataset = x, Model = this, StepsPerExecution = _steps_per_execution }); - Binding.tf_output_redirect.WriteLine($"Testing..."); IEnumerable<(string, Tensor)> logs = null; foreach (var (epoch, iterator) in data_handler.enumerate_epochs()) { @@ -82,22 +79,21 @@ public KeyValuePair[] evaluate(IDatasetV2 x) foreach (var step in data_handler.steps()) { // callbacks.on_train_batch_begin(step) - logs = test_function(iterator); + logs = test_function(data_handler, iterator); } - Binding.tf_output_redirect.WriteLine($"iterator: {epoch + 1}, " + string.Join(", ", logs.Select(x => $"{x.Item1}: {(float)x.Item2}"))); } return logs.Select(x => new KeyValuePair(x.Item1, (float)x.Item2)).ToArray(); } - IEnumerable<(string, Tensor)> test_function(OwnedIterator iterator) + IEnumerable<(string, Tensor)> test_function(DataHandler data_handler, OwnedIterator iterator) { var data = iterator.next(); - var outputs = test_step(data[0], data[1]); + var outputs = test_step(data_handler, data[0], data[1]); tf_with(ops.control_dependencies(new object[0]), ctl => _test_counter.assign_add(1)); return outputs; } - List<(string, Tensor)> test_step(Tensor x, Tensor y) + List<(string, Tensor)> test_step(DataHandler data_handler, Tensor x, Tensor y) { (x, y) = data_handler.DataAdapter.Expand1d(x, y); var y_pred = Apply(x, training: false); diff --git a/src/TensorFlowNET.Keras/Engine/Model.Fit.cs b/src/TensorFlowNET.Keras/Engine/Model.Fit.cs index e0b4af78c..bc2c2cea6 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Fit.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Fit.cs @@ -5,6 +5,8 @@ using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine.DataAdapters; using System.Diagnostics; +using Tensorflow.Keras.Callbacks; +using System.Data; namespace Tensorflow.Keras.Engine { @@ -20,7 +22,7 @@ public partial class Model /// /// /// - public void fit(NDArray x, NDArray y, + public History fit(NDArray x, NDArray y, int batch_size = -1, int epochs = 1, int verbose = 1, @@ -37,7 +39,7 @@ public void fit(NDArray x, NDArray y, var val_x = x[new Slice(train_count)]; var val_y = y[new Slice(train_count)]; - data_handler = new DataHandler(new DataHandlerArgs + var data_handler = new DataHandler(new DataHandlerArgs { X = train_x, Y = train_y, @@ -52,10 +54,10 @@ public void fit(NDArray x, NDArray y, StepsPerExecution = _steps_per_execution }); - FitInternal(epochs, verbose); + return FitInternal(data_handler, epochs, verbose); } - public void fit(IDatasetV2 dataset, + public History fit(IDatasetV2 dataset, IDatasetV2 validation_data = null, int batch_size = -1, int epochs = 1, @@ -67,7 +69,7 @@ public void fit(IDatasetV2 dataset, int workers = 1, bool use_multiprocessing = false) { - data_handler = new DataHandler(new DataHandlerArgs + var data_handler = new DataHandler(new DataHandlerArgs { Dataset = dataset, BatchSize = batch_size, @@ -81,67 +83,52 @@ public void fit(IDatasetV2 dataset, StepsPerExecution = _steps_per_execution }); - FitInternal(epochs, verbose); + return FitInternal(data_handler, epochs, verbose, validation_data: validation_data); } - void FitInternal(int epochs, int verbose) + History FitInternal(DataHandler data_handler, int epochs, int verbose, IDatasetV2 validation_data = null) { stop_training = false; _train_counter.assign(0); - Stopwatch sw = new Stopwatch(); + var callbacks = new CallbackList(new CallbackParams + { + Model = this, + Verbose = verbose, + Epochs = epochs, + Steps = data_handler.Inferredsteps + }); + callbacks.on_train_begin(); + foreach (var (epoch, iterator) in data_handler.enumerate_epochs()) { reset_metrics(); - on_epoch_begin(epoch, epochs); + callbacks.on_epoch_begin(epoch); // data_handler.catch_stop_iteration(); + var logs = new Dictionary(); foreach (var step in data_handler.steps()) { - sw.Start(); - var results = train_step_function(iterator); - sw.Stop(); - on_train_batch_begin(verbose, step, sw.ElapsedMilliseconds, results); + callbacks.on_train_batch_begin(step); + logs = train_step_function(data_handler, iterator); + var end_step = step + data_handler.StepIncrement; + callbacks.on_train_batch_end(end_step, logs); + } - // recycle memory more frequency - if (sw.ElapsedMilliseconds > 100) + if (validation_data != null) + { + var val_logs = evaluate(validation_data); + foreach(var log in val_logs) { - GC.Collect(); + logs["val_" + log.Key] = log.Value; } - sw.Reset(); } - Console.WriteLine(); + + callbacks.on_epoch_end(epoch, logs); GC.Collect(); GC.WaitForPendingFinalizers(); } - } - - void on_epoch_begin(int epoch, int epochs) - { - Binding.tf_output_redirect.WriteLine($"Epoch: {epoch + 1:D3}/{epochs:D3}"); - } - - void on_train_batch_begin(int verbose, long step, long elapse, IEnumerable<(string, Tensor)> results) - { - if (verbose == 1) - { - var result_pairs = string.Join(", ", results.Select(x => $"{x.Item1}: {(float)x.Item2:F6}")); - var progress = ""; - for (int i = 0; i < step + 1; i++) - for (int j = 0; j < 30 / data_handler.Inferredsteps; j++) - progress += "="; - progress += ">"; - - var remaining = ""; - for (int i = 1; i < 30 - progress.Length; i++) - remaining += "."; - - Binding.tf_output_redirect.Write($"{step + 1:D4}/{data_handler.Inferredsteps:D4} [{progress}{remaining}] - {elapse}ms/step {result_pairs}"); - if (!Console.IsOutputRedirected) - { - Console.CursorLeft = 0; - } - } + return callbacks.History; } } } diff --git a/src/TensorFlowNET.Keras/Engine/Model.Train.cs b/src/TensorFlowNET.Keras/Engine/Model.Train.cs index f2ff68e97..0090b69e7 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Train.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Train.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using System.Linq; using Tensorflow.Gradients; +using Tensorflow.Keras.Engine.DataAdapters; using Tensorflow.Keras.Optimizers; using static Tensorflow.Binding; @@ -8,10 +9,10 @@ namespace Tensorflow.Keras.Engine { public partial class Model { - IEnumerable<(string, Tensor)> train_step_function(OwnedIterator iterator) + Dictionary train_step_function(DataHandler data_handler, OwnedIterator iterator) { var data = iterator.next(); - var outputs = train_step(data[0], data[1]); + var outputs = train_step(data_handler, data[0], data[1]); tf_with(ops.control_dependencies(new object[0]), ctl => _train_counter.assign_add(1)); return outputs; } @@ -21,7 +22,7 @@ public partial class Model /// /// /// - List<(string, Tensor)> train_step(Tensor x, Tensor y) + Dictionary train_step(DataHandler data_handler, Tensor x, Tensor y) { (x, y) = data_handler.DataAdapter.Expand1d(x, y); using var tape = tf.GradientTape(); @@ -37,7 +38,9 @@ public partial class Model _minimize(tape, optimizer, loss, TrainableVariables); compiled_metrics.update_state(y, y_pred); - return metrics.Select(x => (x.Name, x.result())).ToList(); + var dict = new Dictionary(); + metrics.ToList().ForEach(x => dict[x.Name] = (float)x.result()); + return dict; } void _minimize(GradientTape tape, OptimizerV2 optimizer, Tensor loss, List trainable_variables) diff --git a/src/TensorFlowNET.Keras/Engine/Model.cs b/src/TensorFlowNET.Keras/Engine/Model.cs index 162d06c57..9bab9bd2f 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.cs @@ -34,7 +34,6 @@ public partial class Model : Layer, IModel IVariableV1 _predict_counter; bool _base_model_initialized; bool stop_training; - DataHandler data_handler; public Model(ModelArgs args) : base(args) diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs index d4ac4b905..029592c3f 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs @@ -147,7 +147,7 @@ public void EinsumDense() Assert.AreEqual(expected_output, actual_output); } - [TestMethod] + [TestMethod, Ignore("WIP")] public void SimpleRNN() { tf.UseKeras(); From 43625abe917a4712e8cdad9c7b49c9875f302a68 Mon Sep 17 00:00:00 2001 From: Superpiffer Date: Wed, 1 Feb 2023 16:52:49 +0100 Subject: [PATCH 18/52] Removed use of tf.Status static instance In multithreading .NET 4.8 applications, sometimes in Session finalizer the method c_api.TF_DeleteSession find f.Status static instance already disposed for some reason. No problem for .NET 6 application or with a single thread. --- src/TensorFlowNET.Core/Sessions/BaseSession.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TensorFlowNET.Core/Sessions/BaseSession.cs b/src/TensorFlowNET.Core/Sessions/BaseSession.cs index 1c9ed2a01..0051a6b33 100644 --- a/src/TensorFlowNET.Core/Sessions/BaseSession.cs +++ b/src/TensorFlowNET.Core/Sessions/BaseSession.cs @@ -291,7 +291,7 @@ private void _extend_graph() protected override void DisposeUnmanagedResources(IntPtr handle) { // c_api.TF_CloseSession(handle, tf.Status.Handle); - c_api.TF_DeleteSession(handle, tf.Status.Handle); + c_api.TF_DeleteSession(handle, c_api.TF_NewStatus()); } } } From 197224fd747c37b12d9468f0b589b4a1681aee8c Mon Sep 17 00:00:00 2001 From: Haiping Date: Sat, 4 Feb 2023 11:58:06 -0600 Subject: [PATCH 19/52] Add pb model save (#976) * Add check for dims of x and y in model.fit. * Init the serialization of keras pb model. * Add more facilities to the saved model framework. * Add ListWrapper and ITrackable, and revise implmentations. * Add serialized attributes. * Implement layer serializations. * Add lacked implementations (mainly MultiDeviceSaver). * Support autograph.to_graph under graph mode. * Add more implementations to the pb model save. * Add more implementations to the keras part of pb model save. * Refine some code after merge. * Add two simple sequential test case of pb model save. * Implement serializing attributes other keras arg definitions. * Add alexnet pb save test. * Check and refine the code. --------- Co-authored-by: AsakusaRinne --- src/TensorFlowNET.Core/APIs/tf.compat.cs | 22 + .../Checkpoint/CheckPointUtils.cs | 152 +++++ .../Checkpoint/CheckpointOptions.cs | 5 + .../Checkpoint/ObjectGraphView.cs | 64 +++ src/TensorFlowNET.Core/Checkpoint/SaveUtil.cs | 255 +++++++++ .../Checkpoint/SaveUtilV1.cs | 223 ++++++++ .../Checkpoint/SaveableCompat.cs | 16 + .../Checkpoint/TrackableView.cs | 82 +++ .../Checkpoint/checkpoint.cs | 195 +++++++ .../Checkpoint/functional_saver.cs | 540 ++++++++++++++++++ src/TensorFlowNET.Core/DisposableObject.cs | 70 ++- src/TensorFlowNET.Core/Eager/execute.cs | 31 + .../Exceptions/AssertionError.cs | 14 + .../Framework/meta_graph.cs | 86 ++- .../Functions/ConcreteFunction.cs | 3 +- src/TensorFlowNET.Core/Functions/Function.cs | 11 +- src/TensorFlowNET.Core/Graphs/AutoGraph.cs | 46 +- .../ArgsDefinition/Activation/ELUArgs.cs | 11 +- .../Activation/LeakyReLuArgs.cs | 6 +- .../ArgsDefinition/Activation/SoftmaxArgs.cs | 11 +- .../ArgsDefinition/Attention/AttentionArgs.cs | 4 + .../Attention/BaseDenseAttentionArgs.cs | 5 +- .../Attention/MultiHeadAttentionArgs.cs | 20 +- .../ArgsDefinition/AutoSerializeLayerArgs.cs | 25 + .../Convolution/ConvolutionalArgs.cs | 40 +- .../Keras/ArgsDefinition/Core/DenseArgs.cs | 42 +- .../{Attention => Core}/EinsumDenseArgs.cs | 36 +- .../ArgsDefinition/Core/EmbeddingArgs.cs | 15 +- .../ArgsDefinition/Core/InputLayerArgs.cs | 17 +- .../ArgsDefinition/Cropping/Cropping2DArgs.cs | 16 - .../ArgsDefinition/Cropping/Cropping3DArgs.cs | 16 - .../ArgsDefinition/Cropping/CroppingArgs.cs | 10 - .../Keras/ArgsDefinition/DataAdapterArgs.cs | 3 +- .../Keras/ArgsDefinition/DataHandlerArgs.cs | 3 +- .../Keras/ArgsDefinition/LayerArgs.cs | 31 +- .../Keras/ArgsDefinition/Lstm/LSTMCellArgs.cs | 6 - .../Keras/ArgsDefinition/Merging/MergeArgs.cs | 1 + .../Keras/ArgsDefinition/NodeArgs.cs | 6 +- .../Normalization/BatchNormalizationArgs.cs | 20 +- .../Normalization/LayerNormalizationArgs.cs | 15 +- .../Keras/ArgsDefinition/OptimizerV2Args.cs | 6 +- .../ArgsDefinition/Pooling/Pooling1DArgs.cs | 10 +- .../ArgsDefinition/Pooling/Pooling2DArgs.cs | 10 +- .../Preprocessing/PreprocessingLayerArgs.cs | 2 +- .../Preprocessing/RescalingArgs.cs | 12 + .../Preprocessing/ResizingArgs.cs | 1 + .../Preprocessing/TextVectorizationArgs.cs | 11 +- .../Regularization/DropoutArgs.cs | 9 +- .../ArgsDefinition/Rescaling/RescalingArgs.cs | 8 - .../Reshaping/Cropping2DArgs.cs | 18 + .../Reshaping/Cropping3DArgs.cs | 18 + .../ArgsDefinition/Reshaping/CroppingArgs.cs | 12 + .../ArgsDefinition/Reshaping/FlattenArgs.cs | 7 +- .../ArgsDefinition/Reshaping/PermuteArgs.cs | 12 +- .../ArgsDefinition/Reshaping/ReshapeArgs.cs | 7 +- .../Reshaping/UpSampling2DArgs.cs | 9 +- .../Reshaping/ZeroPadding2DArgs.cs | 1 + .../ArgsDefinition/{Lstm => Rnn}/LSTMArgs.cs | 5 +- .../Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs | 7 + .../Keras/ArgsDefinition/Rnn/RNNArgs.cs | 15 +- .../CustomizedActivationJsonConverter.cs | 50 ++ .../Common/CustomizedAxisJsonConverter.cs | 48 ++ .../CustomizedNodeConfigJsonConverter.cs | 73 +++ .../Common/CustomizedShapeJsonConverter.cs | 67 +++ .../Keras/Engine/InputSpec.cs | 31 +- src/TensorFlowNET.Core/Keras/Layers/ILayer.cs | 6 +- .../Keras/Layers/ILayersApi.Cropping.cs | 2 +- .../Keras/Saving/IKerasConfig.cs | 15 + .../Keras/Saving/LayerConfig.cs | 9 +- .../Keras/Saving/ModelConfig.cs | 9 +- .../Keras/Saving/NodeConfig.cs | 7 +- .../SavedModel/ISerializedAttributes.cs | 35 ++ .../Keras/Saving/TensorShapeConfig.cs | 21 + .../ModelSaving/SaveOptions.cs | 44 +- src/TensorFlowNET.Core/NumPy/Axis.cs | 11 +- src/TensorFlowNET.Core/Numpy/Shape.cs | 3 + .../Operations/Initializers/Constant.cs | 10 + .../Operations/Initializers/GlorotUniform.cs | 10 +- .../Operations/Initializers/IInitializer.cs | 7 + .../Operations/Initializers/Ones.cs | 7 + .../Operations/Initializers/Orthogonal.cs | 7 +- .../Operations/Initializers/RandomNormal.cs | 12 + .../Operations/Initializers/RandomUniform.cs | 12 + .../Initializers/TruncatedNormal.cs | 11 + .../Initializers/VarianceScaling.cs | 13 + .../Operations/Initializers/Zeros.cs | 5 + .../Operations/NnOps/RNNCell.cs | 8 +- src/TensorFlowNET.Core/Operations/gen_ops.cs | 59 +- src/TensorFlowNET.Core/Operations/io_ops.cs | 32 ++ .../Operations/resource_variable_ops.cs | 60 ++ .../Protobuf/SavedObjectGraph.cs | 10 +- .../Protobuf/TrackableObjectGraph.cs | 16 + .../Tensorflow.Binding.csproj | 1 + src/TensorFlowNET.Core/Tensors/dtypes.cs | 18 + .../Training/AutoTrackable.cs | 69 ++- .../Training/IWithTrackable.cs | 12 + src/TensorFlowNET.Core/Training/LayerUtils.cs | 9 + src/TensorFlowNET.Core/Training/Optimizer.cs | 7 +- .../Saving/ResourceVariableSaveable.cs | 28 + .../Training/Saving/SaveSpec.cs | 2 +- .../Training/Saving/SaveableObject.cs | 38 +- .../Training/Saving/SavedModel/AssetInfo.cs | 11 + .../Saving/SavedModel/AugmentedGraphView.cs | 133 +++++ .../Training/Saving/SavedModel/Constants.cs | 33 ++ .../Saving/SavedModel/RevivedTypes.cs | 17 + .../Training/Saving/SavedModel/SaveType.cs | 9 + .../Saving/SavedModel/SaveableView.cs | 299 ++++++++++ .../Saving/SavedModel/TagConstants.cs | 10 + .../Training/Saving/SavedModel/builder.cs | 22 + .../Training/Saving/SavedModel/save.cs | 269 +++++++++ .../Saving/SavedModel/save_context.cs | 53 ++ .../SavedModel/signature_serialization.cs | 107 ++++ .../Training/Saving/SavedModel/utils.cs | 57 ++ .../Saving/saveable_object_util.py.cs | 255 ++++++++- src/TensorFlowNET.Core/Training/Trackable.cs | 192 ++++++- .../Training/TrackableUtils.cs | 172 ++++++ .../Training/data_structures.cs | 370 ++++++++++++ .../Variables/BaseResourceVariable.cs | 74 ++- .../Variables/IVariableV1.cs | 1 + .../Variables/RefVariable.cs | 4 +- .../Variables/ResourceVariable.cs | 3 + .../Variables/UninitializedVariable.cs | 70 +++ src/TensorFlowNET.Core/ops.cs | 18 + src/TensorFlowNET.Keras/Activations.cs | 82 +++ .../Activations/Activations.Linear.cs | 10 - .../Activations/Activations.Relu.cs | 10 - .../Activations/Activations.Sigmoid.cs | 11 - .../Activations/Activations.Softmax.cs | 11 - .../Activations/Activations.Tanh.cs | 11 - .../Engine/Functional.GetConfig.cs | 31 +- src/TensorFlowNET.Keras/Engine/Functional.cs | 50 ++ .../Engine/Layer.Serialize.cs | 32 ++ src/TensorFlowNET.Keras/Engine/Layer.cs | 36 +- src/TensorFlowNET.Keras/Engine/Model.Fit.cs | 5 + src/TensorFlowNET.Keras/Engine/Model.Save.cs | 19 +- src/TensorFlowNET.Keras/Engine/Model.cs | 19 + .../Layers/Activation/ELU.cs | 1 + .../Layers/Activation/Exponential.cs | 1 + .../Layers/Activation/SELU.cs | 9 +- .../Layers/Attention/Attention.cs | 3 +- .../Layers/Attention/BaseDenseAttention.cs | 3 +- .../Layers/Attention/MultiHeadAttention.cs | 1 + .../Layers/Convolution/Conv2DTranspose.cs | 1 + .../Layers/Convolution/Convolutional.cs | 1 + src/TensorFlowNET.Keras/Layers/Core/Dense.cs | 1 + .../Layers/Core/EinsumDense.cs | 2 +- .../Layers/Core/Embedding.cs | 1 + .../Layers/Core/InputLayer.cs | 3 + .../Layers/Cropping/Cropping2D.cs | 113 ---- .../Layers/Cropping/Cropping3D.cs | 123 ---- .../Layers/LayersApi.Cropping.cs | 10 +- src/TensorFlowNET.Keras/Layers/LayersApi.cs | 39 +- .../Layers/Merging/Concatenate.cs | 1 + .../Layers/Merging/Merge.cs | 1 + .../Normalization/BatchNormalization.cs | 3 +- .../Normalization/LayerNormalization.cs | 1 + .../{Rescaling => Preprocessing}/Rescaling.cs | 0 .../{Cropping => Reshaping}/Cropping1D.cs | 16 +- .../Layers/Reshaping/Cropping2D.cs | 140 +++++ .../Layers/Reshaping/Cropping3D.cs | 150 +++++ .../Layers/Reshaping/Permute.cs | 1 + .../Layers/{Lstm => Rnn}/LSTM.cs | 5 +- .../Layers/{Lstm => Rnn}/LSTMCell.cs | 4 +- src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs | 1 - .../Layers/Rnn/SimpleRNN.cs | 14 +- .../Layers/Rnn/StackedRNNCells.cs | 3 +- .../Protobuf/SavedMetadata.cs | 12 + src/TensorFlowNET.Keras/Protobuf/Versions.cs | 7 + .../Saving/SavedModel/Constants.cs | 41 ++ .../Saving/SavedModel/Save.cs | 162 ++++++ .../Saving/SavedModel/base_serialization.cs | 37 ++ .../Saving/SavedModel/layer_serialization.cs | 165 ++++++ .../SavedModel/serialized_attributes.cs | 282 +++++++++ .../Saving/SavedModel/utils.cs | 47 ++ .../Saving/TensorShapeConfig.cs | 15 - .../Saving/serialization.cs | 125 ++++ .../Utils/base_layer_utils.cs | 2 +- .../Utils/generic_utils.cs | 23 +- .../InitializerTest.cs | 4 +- .../Layers/ModelSaveTest.cs | 5 +- .../SaveModel/SequentialModelTest.cs | 202 +++++++ 181 files changed, 6968 insertions(+), 567 deletions(-) create mode 100644 src/TensorFlowNET.Core/Checkpoint/CheckPointUtils.cs create mode 100644 src/TensorFlowNET.Core/Checkpoint/CheckpointOptions.cs create mode 100644 src/TensorFlowNET.Core/Checkpoint/ObjectGraphView.cs create mode 100644 src/TensorFlowNET.Core/Checkpoint/SaveUtil.cs create mode 100644 src/TensorFlowNET.Core/Checkpoint/SaveUtilV1.cs create mode 100644 src/TensorFlowNET.Core/Checkpoint/SaveableCompat.cs create mode 100644 src/TensorFlowNET.Core/Checkpoint/TrackableView.cs create mode 100644 src/TensorFlowNET.Core/Checkpoint/checkpoint.cs create mode 100644 src/TensorFlowNET.Core/Checkpoint/functional_saver.cs create mode 100644 src/TensorFlowNET.Core/Eager/execute.cs create mode 100644 src/TensorFlowNET.Core/Exceptions/AssertionError.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/AutoSerializeLayerArgs.cs rename src/TensorFlowNET.Core/Keras/ArgsDefinition/{Attention => Core}/EinsumDenseArgs.cs (65%) delete mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Cropping/Cropping2DArgs.cs delete mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Cropping/Cropping3DArgs.cs delete mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Cropping/CroppingArgs.cs delete mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Lstm/LSTMCellArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Preprocessing/RescalingArgs.cs delete mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Rescaling/RescalingArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/Cropping2DArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/Cropping3DArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/CroppingArgs.cs rename src/TensorFlowNET.Core/Keras/ArgsDefinition/{Lstm => Rnn}/LSTMArgs.cs (67%) create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs create mode 100644 src/TensorFlowNET.Core/Keras/Common/CustomizedActivationJsonConverter.cs create mode 100644 src/TensorFlowNET.Core/Keras/Common/CustomizedAxisJsonConverter.cs create mode 100644 src/TensorFlowNET.Core/Keras/Common/CustomizedNodeConfigJsonConverter.cs create mode 100644 src/TensorFlowNET.Core/Keras/Common/CustomizedShapeJsonConverter.cs create mode 100644 src/TensorFlowNET.Core/Keras/Saving/IKerasConfig.cs create mode 100644 src/TensorFlowNET.Core/Keras/Saving/SavedModel/ISerializedAttributes.cs create mode 100644 src/TensorFlowNET.Core/Keras/Saving/TensorShapeConfig.cs create mode 100644 src/TensorFlowNET.Core/Training/IWithTrackable.cs create mode 100644 src/TensorFlowNET.Core/Training/LayerUtils.cs create mode 100644 src/TensorFlowNET.Core/Training/Saving/SavedModel/AssetInfo.cs create mode 100644 src/TensorFlowNET.Core/Training/Saving/SavedModel/AugmentedGraphView.cs create mode 100644 src/TensorFlowNET.Core/Training/Saving/SavedModel/Constants.cs create mode 100644 src/TensorFlowNET.Core/Training/Saving/SavedModel/RevivedTypes.cs create mode 100644 src/TensorFlowNET.Core/Training/Saving/SavedModel/SaveType.cs create mode 100644 src/TensorFlowNET.Core/Training/Saving/SavedModel/SaveableView.cs create mode 100644 src/TensorFlowNET.Core/Training/Saving/SavedModel/TagConstants.cs create mode 100644 src/TensorFlowNET.Core/Training/Saving/SavedModel/builder.cs create mode 100644 src/TensorFlowNET.Core/Training/Saving/SavedModel/save.cs create mode 100644 src/TensorFlowNET.Core/Training/Saving/SavedModel/save_context.cs create mode 100644 src/TensorFlowNET.Core/Training/Saving/SavedModel/signature_serialization.cs create mode 100644 src/TensorFlowNET.Core/Training/Saving/SavedModel/utils.cs create mode 100644 src/TensorFlowNET.Core/Training/TrackableUtils.cs create mode 100644 src/TensorFlowNET.Core/Training/data_structures.cs create mode 100644 src/TensorFlowNET.Core/Variables/UninitializedVariable.cs create mode 100644 src/TensorFlowNET.Keras/Activations.cs delete mode 100644 src/TensorFlowNET.Keras/Activations/Activations.Linear.cs delete mode 100644 src/TensorFlowNET.Keras/Activations/Activations.Relu.cs delete mode 100644 src/TensorFlowNET.Keras/Activations/Activations.Sigmoid.cs delete mode 100644 src/TensorFlowNET.Keras/Activations/Activations.Softmax.cs delete mode 100644 src/TensorFlowNET.Keras/Activations/Activations.Tanh.cs create mode 100644 src/TensorFlowNET.Keras/Engine/Layer.Serialize.cs delete mode 100644 src/TensorFlowNET.Keras/Layers/Cropping/Cropping2D.cs delete mode 100644 src/TensorFlowNET.Keras/Layers/Cropping/Cropping3D.cs rename src/TensorFlowNET.Keras/Layers/{Rescaling => Preprocessing}/Rescaling.cs (100%) rename src/TensorFlowNET.Keras/Layers/{Cropping => Reshaping}/Cropping1D.cs (77%) create mode 100644 src/TensorFlowNET.Keras/Layers/Reshaping/Cropping2D.cs create mode 100644 src/TensorFlowNET.Keras/Layers/Reshaping/Cropping3D.cs rename src/TensorFlowNET.Keras/Layers/{Lstm => Rnn}/LSTM.cs (87%) rename src/TensorFlowNET.Keras/Layers/{Lstm => Rnn}/LSTMCell.cs (72%) create mode 100644 src/TensorFlowNET.Keras/Saving/SavedModel/Constants.cs create mode 100644 src/TensorFlowNET.Keras/Saving/SavedModel/Save.cs create mode 100644 src/TensorFlowNET.Keras/Saving/SavedModel/base_serialization.cs create mode 100644 src/TensorFlowNET.Keras/Saving/SavedModel/layer_serialization.cs create mode 100644 src/TensorFlowNET.Keras/Saving/SavedModel/serialized_attributes.cs create mode 100644 src/TensorFlowNET.Keras/Saving/SavedModel/utils.cs delete mode 100644 src/TensorFlowNET.Keras/Saving/TensorShapeConfig.cs create mode 100644 src/TensorFlowNET.Keras/Saving/serialization.cs create mode 100644 test/TensorFlowNET.Keras.UnitTest/SaveModel/SequentialModelTest.cs diff --git a/src/TensorFlowNET.Core/APIs/tf.compat.cs b/src/TensorFlowNET.Core/APIs/tf.compat.cs index 4d979eb55..5b2b5a107 100644 --- a/src/TensorFlowNET.Core/APIs/tf.compat.cs +++ b/src/TensorFlowNET.Core/APIs/tf.compat.cs @@ -14,6 +14,8 @@ You may obtain a copy of the License at limitations under the License. ******************************************************************************/ +using System.Text; + namespace Tensorflow { public partial class tensorflow @@ -23,6 +25,26 @@ public partial class tensorflow public class CompatApi { public CompatV1Api v1 { get; } = new CompatV1Api(); + + internal string as_text(string bytes_or_text, Encoding? encoding = null) + { + if(encoding is null) encoding = Encoding.UTF8; + return bytes_or_text; + } + internal string as_text(byte[] bytes_or_text, Encoding? encoding = null) + { + if(encoding is null) encoding = Encoding.UTF8; + return encoding.GetString(bytes_or_text); + } + + internal string as_str(string bytes_or_text, Encoding? encoding = null) + { + return as_text(bytes_or_text, encoding); + } + internal string as_str(byte[] bytes_or_text, Encoding? encoding = null) + { + return as_text(bytes_or_text, encoding); + } } public bool executing_eagerly() diff --git a/src/TensorFlowNET.Core/Checkpoint/CheckPointUtils.cs b/src/TensorFlowNET.Core/Checkpoint/CheckPointUtils.cs new file mode 100644 index 000000000..8ae2dae8f --- /dev/null +++ b/src/TensorFlowNET.Core/Checkpoint/CheckPointUtils.cs @@ -0,0 +1,152 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using Tensorflow.Train; +using Tensorflow.Training; +using pbc = global::Google.Protobuf.Collections; + +namespace Tensorflow.Checkpoint; + +public static class CheckPointUtils +{ + private static string _ESCAPE_CHAR = "."; + public static (IList, IDictionary>, IDictionary, + IDictionary>, + IDictionary) objects_ids_and_slot_variables_and_paths(ObjectGraphView graph_view) + { + var (trackable_objects, node_paths) = graph_view.breadth_first_traversal(); + Dictionary object_names = new(); + foreach (var pair in node_paths) + { + object_names[pair.Key] = TrackableUtils.object_path_to_string(pair.Value); + } + + Dictionary node_ids = new(); + for (int i = 0; i < trackable_objects.Count; i++) + { + node_ids[trackable_objects[i]] = i; + } + + var slot_variables = serialize_slot_variables(trackable_objects, node_ids, object_names); + return (trackable_objects, node_paths, node_ids, slot_variables, object_names); + } + + public static + IDictionary> + serialize_slot_variables(IEnumerable trackable_objects, + IDictionary node_ids, IDictionary object_names) + { + var non_slot_objects = trackable_objects.ToList(); + Dictionary> + slot_variables = new(); + foreach (var trackable in non_slot_objects) + { + if (trackable is not Optimizer) + { + continue; + } + + var optim = (Optimizer)trackable; + var slot_names = optim.get_slot_names(); + foreach (var slot_name in slot_names) + { + for (int original_variable_node_id = 0; + original_variable_node_id < non_slot_objects.Count; + original_variable_node_id++) + { + var original_variable = non_slot_objects[original_variable_node_id]; + IVariableV1 slot_variable; + if (original_variable is not IVariableV1) + { + slot_variable = null; + } + slot_variable = optim.get_slot((IVariableV1)original_variable, slot_name); + if(slot_variable is null) continue; + + // There're some problems about the inherits of `Variable` and `Trackable`. + throw new NotImplementedException(); + } + } + } + + return slot_variables; + } + + public static Trackable get_mapped_trackable(Trackable trackable, IDictionary? object_map) + { + if (object_map is null || !object_map.TryGetValue(trackable, out var possible_res)) + { + return trackable; + } + else + { + return possible_res; + } + } + + public static string get_full_name(Trackable variable) + { + // TODO: This state is not correct, the whole framework need to be updated in the future. + if (!(variable is IVariableV1 || resource_variable_ops.is_resource_variable(variable))) + { + return ""; + } + // skip the check of attribute `_save_slice_info` . + + // TODO: Need to be revised!!! + Debug.Assert(variable is BaseResourceVariable); + return ((BaseResourceVariable)variable).Name; + } + + public static void add_checkpoint_values_check(TrackableObjectGraph object_graph_proto) + { + HashSet checkpointed_trackables = new(); + Dictionary> parents = new(); + for (int i = 0; i < object_graph_proto.Nodes.Count; i++) + { + var object_proto = object_graph_proto.Nodes[i]; + // skip the process of registered saver. + if (object_proto.Attributes is not null && object_proto.Attributes.Count > 0 || + object_proto.SlotVariables is not null && object_proto.SlotVariables.Count > 0) + { + checkpointed_trackables.Add(i); + } + + foreach (var child_proto in object_proto.Children) + { + var child = child_proto.NodeId; + if (!parents.ContainsKey(child)) + { + parents[child] = new HashSet(); + } + + parents[child].Add(i); + } + } + + Queue to_visit = new(checkpointed_trackables.AsEnumerable()); + while (to_visit.Count > 0) + { + var trackable = to_visit.Dequeue(); + if (!parents.ContainsKey(trackable)) continue; + var current_parents = parents[trackable]; + foreach (var parent in current_parents) + { + checkpointed_trackables.Add(parent); + if (parents.ContainsKey(parent)) + { + to_visit.Enqueue(parent); + } + } + parents.Remove(trackable); + } + + // TODO: Complete it after supporting checkpoint. + // for (int i = 0; i < object_graph_proto.Nodes.Count; i++) + // { + // object_graph_proto.Nodes[i].has_checkpoint_values.value = checkpointed_trackables.Contains(i); + // } + } +} diff --git a/src/TensorFlowNET.Core/Checkpoint/CheckpointOptions.cs b/src/TensorFlowNET.Core/Checkpoint/CheckpointOptions.cs new file mode 100644 index 000000000..75b392af8 --- /dev/null +++ b/src/TensorFlowNET.Core/Checkpoint/CheckpointOptions.cs @@ -0,0 +1,5 @@ +namespace Tensorflow.Checkpoint; + +public record class CheckpointOptions( + string? experimental_io_device = null, + bool experimental_enable_async_checkpoint = false); diff --git a/src/TensorFlowNET.Core/Checkpoint/ObjectGraphView.cs b/src/TensorFlowNET.Core/Checkpoint/ObjectGraphView.cs new file mode 100644 index 000000000..f435dd88b --- /dev/null +++ b/src/TensorFlowNET.Core/Checkpoint/ObjectGraphView.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Serilog.Debugging; +using Tensorflow.Keras.Saving.SavedModel; +using Tensorflow.Train; + +namespace Tensorflow.Checkpoint; + +public class ObjectGraphView: TrackableView, ICloneable +{ + protected IEnumerable? _attached_dependencies; + // TODO: attached_dependencies + public ObjectGraphView(Trackable root, IEnumerable? attached_dependencies = null): base(root) + { + _attached_dependencies = attached_dependencies; + } + + public object Clone() + { + // TODO: Implement real deep copy corresponding to tensorflow/python/checkpoint/graph_view.ObjectGraphView.__deepcopy__ + return new ObjectGraphView(Root, _attached_dependencies); + } + + public virtual List list_children(Trackable obj, SaveType save_type = SaveType.CHECKPOINT, IDictionary>? serialization_cache = null) + { + List res = base.children(obj, save_type, serialization_cache) + .Select(x => new TrackableReference(x.Key, x.Value)).ToList(); + // Check the reference, not value. + if (obj == Root && _attached_dependencies is not null) + { + res.AddRange(_attached_dependencies); + } + + return res; + } + + public override IDictionary children(Trackable obj, SaveType save_type = SaveType.CHECKPOINT, IDictionary>? serialization_cache = null) + { + return list_children(obj, save_type, serialization_cache).ToDictionary(x => x.Name, x => x.Refer); + } + + public IEnumerable? AttachedDependencies + { + get => _attached_dependencies; + } + + public virtual (IList, IDictionary>) breadth_first_traversal() + { + return base._descendants_with_paths(); + } + + // TODO: complete the implementation + public void serialize_object_graph(object? saveables_cache = null) + { + throw new NotImplementedException(); + } + + // TODO: complete the implementation + public void frozen_saveable_objects(object? object_map = null, object? to_graph = null, object call_with_mapped_captures = null) + { + throw new NotImplementedException(); + } +} diff --git a/src/TensorFlowNET.Core/Checkpoint/SaveUtil.cs b/src/TensorFlowNET.Core/Checkpoint/SaveUtil.cs new file mode 100644 index 000000000..c54cc93f6 --- /dev/null +++ b/src/TensorFlowNET.Core/Checkpoint/SaveUtil.cs @@ -0,0 +1,255 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using Tensorflow.Train; +using Tensorflow.Training; +using pbc = global::Google.Protobuf.Collections; + +namespace Tensorflow.Checkpoint +{ + internal record class TrackableData( + // A trackable in the root Trackable object graph. + Trackable trackable, + // The index at which the Trackable appears in TrackableObjectGraph.nodes. + int node_id, + // The BFS-generated path from the root object / used to generate readable checkpoint keys. + string object_name, + // A list of ObjectReference for each child connected to this Trackable. + pbc::RepeatedField children_proto, + // A list of SlotVariableReference to save to the object (only valid for Optimizer objects). + pbc::RepeatedField slot_variable_proto, + // The object to save to checkpoint. Usually this is the same as `trackable`, + // but can differ when the the caller wants to specify a different object to + // save. For example, when saving checkpoints asynchronously, variables are + // copied to the CPU. `object_to_save` is set as the copied variable. + Trackable object_to_save + ); + public static class SaveUtil + { + public static (IDictionary>>>, IDictionary, IDictionary>, TrackableObjectGraph) + serialize_graph_view(ObjectGraphView graph_view, IDictionary? object_map = null, bool call_with_mapped_captures = false, object? cache = null) + { + var (trackable_data, node_ids) = gather_trackable_data(graph_view, object_map); + var (tensor_trackables, pystate_trackables, registered_trackables) = split_trackables(trackable_data); + + var object_graph_proto = fill_object_graph_proto(trackable_data); + + var serialized_tensors = get_and_write_tensors_to_serialize(tensor_trackables, node_ids, call_with_mapped_captures, cache, object_graph_proto); + var registered_savers = get_and_write_registered_savers(registered_trackables, object_graph_proto); + + Dictionary feed_additions; + if(cache is null) + { + feed_additions = null; + serialized_tensors = serialized_tensors.Concat(get_and_write_tensors_to_serialize(pystate_trackables, node_ids, call_with_mapped_captures, + cache, object_graph_proto)).ToDictionary(x => x.Key, x => x.Value); + } + else + { + feed_additions = null; + // TODO: deal with cache. + throw new NotFiniteNumberException(); + } + + CheckPointUtils.add_checkpoint_values_check(object_graph_proto); + + return (serialized_tensors, feed_additions, registered_savers, object_graph_proto); + } + + private static (IList, IDictionary) gather_trackable_data(ObjectGraphView graph_view, IDictionary? object_map) + { + var (trackable_objects, node_paths) = graph_view.breadth_first_traversal(); + Dictionary object_names = new(); + foreach(var pair in node_paths) + { + object_names[pair.Key] = TrackableUtils.object_path_to_string(pair.Value); + } + Dictionary node_ids = new(); + for(int i = 0; i < trackable_objects.Count; i++) + { + node_ids[trackable_objects[i]] = i; + } + var slot_variables = CheckPointUtils.serialize_slot_variables(trackable_objects, node_ids, object_names); + List trackable_data = new(); + foreach(var trackable in trackable_objects) + { + pbc::RepeatedField children_proto = new(); + foreach(var child in graph_view.list_children(trackable)) + { + children_proto.Add(new TrackableObjectGraph.Types.TrackableObject.Types.ObjectReference() + { + NodeId = node_ids[child.Refer], + LocalName = child.Name + }); + } + slot_variables.TryGetValue(trackable, out var slot_variable); + trackable_data.Add(new TrackableData( + trackable: trackable, + node_id: node_ids[trackable], + object_name: object_names[trackable], + children_proto: children_proto, + slot_variable_proto: slot_variable??new pbc.RepeatedField(), + object_to_save: CheckPointUtils.get_mapped_trackable(trackable, object_map) + )); + } + return (trackable_data, node_ids); + } + + private static TrackableObjectGraph fill_object_graph_proto(IList trackable_data) + { + TrackableObjectGraph object_graph_proto = new(); + for(int i = 0; i < trackable_data.Count; i++) + { + var td = trackable_data[i]; + Debug.Assert(td.node_id == i); + object_graph_proto.Nodes.Add(new TrackableObjectGraph.Types.TrackableObject(td.slot_variable_proto, td.children_proto)); + } + return object_graph_proto; + } + + /// + /// Creates dictionary of tensors to checkpoint, and updates the proto. + /// + /// + /// + /// + /// + /// + private static IDictionary>>> get_and_write_tensors_to_serialize(IList tensor_trackables, IDictionary node_ids, + bool call_with_mapped_captures, object? cache, TrackableObjectGraph object_graph_proto) + { + Dictionary>>> serialized_tensors = new(); + foreach(var td in tensor_trackables) + { + // TODO: deal with cache. + var legacy_name = SaveableCompat.get_saveable_name(td.object_to_save) ?? ""; + Trackable trackable = null; + IDictionary>> tensor_dict; + if(!saveable_object_util.trackable_has_serialize_to_tensor(td.object_to_save) || legacy_name.Length > 0) + { + (trackable, tensor_dict) = get_tensors_from_legacy_saveable(td, node_ids, call_with_mapped_captures, object_graph_proto); + } + else + { + tensor_dict = get_tensors_from_trackable(td, call_with_mapped_captures, object_graph_proto); + trackable = td.object_to_save; + } + if(trackable is not null) + { + serialized_tensors[trackable] = tensor_dict; + } + else + { + serialized_tensors[Trackable.None] = tensor_dict; + } + } + return serialized_tensors; + } + + private static IDictionary>> get_tensors_from_trackable(TrackableData trackable_data, bool call_with_mapped_captures, TrackableObjectGraph object_graph_proto) + { + var trackable = trackable_data.object_to_save; + + // TODO: complete it. Note that actually `call_with_mapped_captures` is of function type. + IDictionary>> ret_tensor_dict; + if (call_with_mapped_captures) + { + throw new NotImplementedException(); + } + else + { + ret_tensor_dict = trackable.serialize_to_tensors(); + } + + // TODO: deal with the type `SaveSpce` (currently it will never be it). + Dictionary>> tensor_dict = new(); + foreach(var pair in ret_tensor_dict) + { + var local_name = TrackableUtils.escape_local_name(pair.Key); + var maybe_tensor = pair.Value; + var checkpoint_key = TrackableUtils.checkpoint_key(trackable_data.object_name, local_name); + + tensor_dict[checkpoint_key] = maybe_tensor; + + if(maybe_tensor.IsTypeOrDeriveFrom()) + { + throw new NotImplementedException(); + //((SaveSpec)maybe_tensor).name = local_name + ((SaveSpec)maybe_tensor).name; + } + + if(object_graph_proto is not null) + { + object_graph_proto.Nodes[trackable_data.node_id].Attributes.Add(new TrackableObjectGraph.Types.TrackableObject.Types.SerializedTensor() + { + Name = local_name, + CheckpointKey = checkpoint_key, + FullName = CheckPointUtils.get_full_name(trackable) + }); + } + } + return tensor_dict; + } + + /// + /// Gets tensors to serialize from a Trackable with legacy SaveableObjects. + /// + /// + /// + /// + /// + /// + private static (Trackable, IDictionary>>) get_tensors_from_legacy_saveable(TrackableData trackable_data, IDictionary node_ids, + bool call_with_mapped_captures, TrackableObjectGraph object_graph_proto) + { + Dictionary object_names = new(); + object_names[trackable_data.trackable] = trackable_data.object_name; + Dictionary object_map = new(); + object_map[trackable_data.trackable] = trackable_data.object_to_save; + + var (checkpoint_factory_map, _) = SaveUtilV1.get_checkpoint_factories_and_keys(object_names, object_map); + var (named_saveable_objects, _) = SaveUtilV1.generate_saveable_objects(checkpoint_factory_map, object_graph_proto, node_ids, object_map, + call_with_mapped_captures, saveables_cache: null); + var trackable = new SaveableCompatibilityConverter(trackable_data.object_to_save, named_saveable_objects); + return (trackable, trackable.serialize_to_tensors()); + } + + private static IDictionary> get_and_write_registered_savers(IDictionary> registered_trackables, TrackableObjectGraph object_graph_proto) + { + Dictionary> registered_savers = new(); + foreach(var pair in registered_trackables) + { + foreach(var td in pair.Value) + { + if (registered_savers.ContainsKey(pair.Key)) + { + registered_savers[pair.Key] = new Dictionary(); + } + else + { + registered_savers[pair.Key][td.object_name] = td.object_to_save; + } + + var object_proto = object_graph_proto.Nodes[td.node_id]; + // TODO: add APIs and complete it. Now the `TrackableObjectGraph.Types.TrackableObject` lacks `registered_savers`. + } + } + return registered_savers; + } + + private static (IList, IList, IDictionary>) split_trackables(IEnumerable trackable_data) + { + List tensor_trackables = new(); + List py_state_trackables = new(); // skip the process of `PyState` for the lack of API. This is only a pleceholder. + Dictionary> registered_trackables = new(); + + foreach(var td in trackable_data) + { + // TODO: deal with registration. + tensor_trackables.Add(td); + } + return (tensor_trackables, py_state_trackables, registered_trackables); + } + } +} diff --git a/src/TensorFlowNET.Core/Checkpoint/SaveUtilV1.cs b/src/TensorFlowNET.Core/Checkpoint/SaveUtilV1.cs new file mode 100644 index 000000000..3267ae126 --- /dev/null +++ b/src/TensorFlowNET.Core/Checkpoint/SaveUtilV1.cs @@ -0,0 +1,223 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Tensorflow.Exceptions; +using Tensorflow.Train; +using Tensorflow.Training; +using pbc = global::Google.Protobuf.Collections; +using static Tensorflow.Binding; +using Google.Protobuf; + +namespace Tensorflow.Checkpoint; + +public static class SaveUtilV1 +{ + public static (IDictionary>, object?) get_checkpoint_factories_and_keys(IDictionary object_names, + IDictionary? object_map = null) + { + // According to https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/registration/README.md, + // till now only internal registrations are allowed. So, we won't return a saver in this function. + // The implementation of this function should be updated if tensorflow update it. + Dictionary> checkpoint_factory_map = new(); + foreach (var pair in object_names) + { + var trackable = pair.Key; + var object_name = pair.Value; + var object_to_save = CheckPointUtils.get_mapped_trackable(trackable, object_map); + + // skip the registration process. + + List current_list = new(); + foreach (var name_and_factory in saveable_object_util.saveable_objects_from_trackable(object_to_save)) + { + // treat name as key_suffix. + var name = name_and_factory.Key; + var checkpoint_key = TrackableUtils.checkpoint_key(object_name, name); + + current_list.Add(new CheckpointFactoryData(name_and_factory.Value, name, checkpoint_key)); + } + + checkpoint_factory_map[trackable] = current_list; + } + + return (checkpoint_factory_map, null); + } + + public static (IList, IDictionary>?) frozen_saveables_and_savers(ObjectGraphView graph_view, + IDictionary object_map, Graph? to_graph, bool call_with_mapped_captures, + object? saveables_cache = null) + { + if (to_graph is not null) + { + var g = to_graph.as_default(); + var (named_saveable_objects, graph_proto, _, registered_savers) = serialize_gathered_objects(graph_view, + object_map, call_with_mapped_captures, saveables_cache); + tf.device("/cpu:0"); + var object_graph_tensor = constant_op.constant(graph_proto.ToByteArray()); + named_saveable_objects.Add(new NoRestoreSaveable(object_graph_tensor, Trackable.Constants.OBJECT_GRAPH_PROTO_KEY)); + g.Exit(); + return (named_saveable_objects, registered_savers); + } + else + { + using (new ops.NullContextManager()) + { + var (named_saveable_objects, graph_proto, _, registered_savers) = serialize_gathered_objects(graph_view, + object_map, call_with_mapped_captures, saveables_cache); + tf.device("/cpu:0"); + var object_graph_tensor = constant_op.constant(graph_proto.ToString(), TF_DataType.TF_STRING); + named_saveable_objects.Add(new NoRestoreSaveable(object_graph_tensor, Trackable.Constants.OBJECT_GRAPH_PROTO_KEY)); + return (named_saveable_objects, registered_savers); + } + } + } + + public static (IList, TrackableObjectGraph, object?, IDictionary>?) serialize_gathered_objects(ObjectGraphView graph_view, + IDictionary object_map, bool call_with_mapped_captures, object? saveables_cache = null) + { + var (trackable_objects, node_paths) = graph_view.breadth_first_traversal(); + Dictionary object_names = new(); + foreach (var pair in node_paths) + { + object_names[pair.Key] = TrackableUtils.object_path_to_string(pair.Value); + } + + Dictionary node_ids = new(); + for (int i = 0; i < trackable_objects.Count; i++) + { + node_ids[trackable_objects[i]] = i; + } + + var slot_variables = CheckPointUtils.serialize_slot_variables(trackable_objects, node_ids, object_names); + var object_graph_proto = fill_object_graph_proto(graph_view, trackable_objects, node_ids, slot_variables); + var (named_saveable_objects, feed_additions, registered_savers) = add_attributes_to_object_graph( + trackable_objects, object_graph_proto, node_ids, object_names, object_map, call_with_mapped_captures, + saveables_cache); + + CheckPointUtils.add_checkpoint_values_check(object_graph_proto); + return (named_saveable_objects, object_graph_proto, feed_additions, registered_savers); + } + + private static TrackableObjectGraph fill_object_graph_proto(ObjectGraphView graph_view, IList trackable_objects, + IDictionary node_ids, + IDictionary> + slot_variables) + { + TrackableObjectGraph object_graph_proto = new(); + for (int i = 0; i < trackable_objects.Count; i++) + { + var trackable = trackable_objects[i]; + Debug.Assert(node_ids[trackable] == i); + TrackableObjectGraph.Types.TrackableObject object_proto; + if (slot_variables.TryGetValue(trackable, out var slots)) + { + object_proto = new TrackableObjectGraph.Types.TrackableObject(slots); + } + else + { + object_proto = new TrackableObjectGraph.Types.TrackableObject(); + } + object_graph_proto.Nodes.Add(object_proto); + foreach (var child in graph_view.list_children(trackable)) + { + object_proto.Children.Add(new TrackableObjectGraph.Types.TrackableObject.Types.ObjectReference() + { NodeId = node_ids[child.Refer], LocalName = child.Name }); + } + } + + return object_graph_proto; + } + + private static (IList, object?, IDictionary>?) add_attributes_to_object_graph( + IList trackable_objects, + TrackableObjectGraph object_graph_proto, IDictionary node_ids, + IDictionary object_names, IDictionary object_map, + bool call_with_mapped_captures, object? saveables_cache = null) + { + int cnt = Math.Min(trackable_objects.Count, object_graph_proto.Nodes.Count); + for (int i = 0; i < cnt; i++) + { + Debug.Assert(node_ids[trackable_objects[i]] == i); + } + + var (checkpoint_factory_map, unmmaped_registered_savers) = + get_checkpoint_factories_and_keys(object_names, object_map); + + // skip the process of registered savers + + var (named_saveable_objects, feed_additions) = generate_saveable_objects(checkpoint_factory_map, + object_graph_proto, node_ids, object_map, call_with_mapped_captures, saveables_cache); + return (named_saveable_objects, feed_additions, null); + } + + public static (IList, object?) generate_saveable_objects( + IDictionary> checkpoint_factory_map, + TrackableObjectGraph? object_graph_proto, IDictionary? node_ids, + IDictionary object_map, bool call_with_mapped_captures, object? saveables_cache = null) + { + List named_saveable_objects = new(); + foreach (var pair in checkpoint_factory_map) + { + var trackable = pair.Key; + var factory_data_list = pair.Value; + bool fill_object_proto = object_graph_proto is not null && node_ids is not null; + TrackableObjectGraph.Types.TrackableObject object_proto = null!; + if (fill_object_proto) + { + object_proto = object_graph_proto.Nodes[node_ids[trackable]]; + } + + var object_to_save = CheckPointUtils.get_mapped_trackable(trackable, object_map); + // skip cache + + foreach (var factory_data in factory_data_list) + { + var name = factory_data.name; + var key = factory_data.checkpoint_key; + var maybe_saveable = factory_data.factory; + + // TODO: oneflow python has a process with callable `saveable_factory`. + List saveables = new(); + if (maybe_saveable.TryGet(out var s)) + { + saveables.Add(s); + } + else + { + saveables.AddRange(saveable_object_util.saveable_objects_for_op(maybe_saveable.GetValue() as Trackable, key)); + } + + foreach (var saveable in saveables) + { + if (!saveable.name.Contains(key)) + { + throw new AssertionError($"The object {trackable} produced a SaveableObject with name " + + $"'{saveable.name}' for attribute '{name}'. Expected a name" + + $" containing '{key}'."); + } + } + + // skip the process of PythonState + + named_saveable_objects.AddRange(saveables); + + if(!fill_object_proto) continue; + + // skip the process of `TrackableSaveable` because of lack of APIs. + + object_proto!.Attributes.Add(new TrackableObjectGraph.Types.TrackableObject.Types.SerializedTensor() + { Name = name, CheckpointKey = key, FullName = CheckPointUtils.get_full_name(object_to_save) }); + } + } + + return (named_saveable_objects, null); + } +} + +public record class CheckpointFactoryData +( + Maybe factory, + string name, + string checkpoint_key +); diff --git a/src/TensorFlowNET.Core/Checkpoint/SaveableCompat.cs b/src/TensorFlowNET.Core/Checkpoint/SaveableCompat.cs new file mode 100644 index 000000000..fa441d799 --- /dev/null +++ b/src/TensorFlowNET.Core/Checkpoint/SaveableCompat.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Train; + +namespace Tensorflow.Checkpoint +{ + internal static class SaveableCompat + { + public static string? get_saveable_name(Trackable cls_or_obj) + { + // TODO: implement it with Attribute. + return null; + } + } +} diff --git a/src/TensorFlowNET.Core/Checkpoint/TrackableView.cs b/src/TensorFlowNET.Core/Checkpoint/TrackableView.cs new file mode 100644 index 000000000..dab6d5d97 --- /dev/null +++ b/src/TensorFlowNET.Core/Checkpoint/TrackableView.cs @@ -0,0 +1,82 @@ +using System; +using Tensorflow.Train; +using System.Collections.Generic; +using System.IO; +using Tensorflow.Keras.Saving.SavedModel; + +namespace Tensorflow.Checkpoint; + +public class TrackableView +{ + protected WeakReference _root_ref; + public TrackableView(Trackable obj) + { + _root_ref = new WeakReference(obj); + } + + public TrackableView(WeakReference obj) + { + _root_ref = obj; + } + + public virtual IDictionary children(Trackable obj, SaveType save_type = SaveType.CHECKPOINT, IDictionary>? cache = null) + { + obj._maybe_initialize_trackable(); + Dictionary children = new(); + // Note: in python the return type of `Trackable._trackable_children` is not fixed. + // Therefore it uses `convert_to_trackable` to have an extra process. + foreach (var pair in obj._trackable_children(save_type, cache)) + { + children[pair.Key] = pair.Value; + } + return children; + } + + public Trackable Root + { + get + { + if (_root_ref.TryGetTarget(out Trackable res)) + { + return res; + } + else + { + throw new InvalidDataException( + "Cannot get the object from the weak reference. Please consider if a null reference is passed to the constructor."); + } + } + } + + /// + /// Returns a list of all nodes and its paths from self.root using a breadth first traversal. + /// Corresponding to tensorflow/python/checkpoint/trackable_view.Trackable._descendants_with_paths + /// + protected (IList, IDictionary>) _descendants_with_paths() + { + List bfs_sorted = new(); + Queue to_visit = new(); + to_visit.Enqueue(Root); + Dictionary> node_paths = new(); + node_paths[this.Root] = new List(); + while (!to_visit.empty()) + { + var current_trackable = to_visit.Dequeue(); + bfs_sorted.Add(current_trackable); + var children_dict = this.children(current_trackable); + foreach (var name in children_dict.Keys) + { + var dependency = children_dict[name]; + if (!node_paths.ContainsKey(dependency)) + { + var list = new List(node_paths[current_trackable]); + list.Add(new TrackableReference(name, dependency)); + node_paths[dependency] = list; + to_visit.Enqueue(dependency); + } + } + } + + return (bfs_sorted, node_paths); + } +} \ No newline at end of file diff --git a/src/TensorFlowNET.Core/Checkpoint/checkpoint.cs b/src/TensorFlowNET.Core/Checkpoint/checkpoint.cs new file mode 100644 index 000000000..0c2862dac --- /dev/null +++ b/src/TensorFlowNET.Core/Checkpoint/checkpoint.cs @@ -0,0 +1,195 @@ +using Google.Protobuf; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Tensorflow.Contexts; +using Tensorflow.Eager; +using Tensorflow.Train; +using static Tensorflow.TrackableObjectGraph.Types.TrackableObject.Types; +using static Tensorflow.Binding; + +namespace Tensorflow.Checkpoint; + +/// +/// Saves and restores a `Trackable` object and its dependencies. +/// +public class TrackableSaver +{ + private ObjectGraphView _graph_view; + private Tensor _cached_save_operation; + private TrackableObjectGraph _last_save_object_graph; + private Tensor? _object_graph_feed_tensor = null; + private Tensor? _file_prefix_feed_tensor = null; + private Dictionary? _object_map = null; + private object? _cache = null; + public TrackableSaver(ObjectGraphView graph_view) + { + _graph_view = graph_view; + + // TODO: cache when not executing eagerly. + // including `_cache`, `_file_prefix_feed_tensor`, `_file_prefix_placeholder`, + // `_object_graph_feed_tensor`, `_object_map`, `_restore_op_cache`, `_saveables_cache` + + } + + private (IDictionary>>>, IDictionary, IDictionary>, TrackableObjectGraph) + gather_serialized_tensors(Tensor? object_graph_tensor = null) + { + var (serialized_tensors, feed_additions, registered_savers, graph_proto) = SaveUtil.serialize_graph_view(_graph_view, _object_map, cache:_cache); + + // TODO: cache. + + if(object_graph_tensor is null) + { + tf.device("/cpu:0"); + object_graph_tensor = constant_op.constant(graph_proto.ToByteArray()); + } + else + { + feed_additions[object_graph_tensor] = graph_proto.ToByteArray(); + } + Debug.Assert(!serialized_tensors.ContainsKey(Trackable.None) || !serialized_tensors[Trackable.None].ContainsKey(Trackable.Constants.OBJECT_GRAPH_PROTO_KEY)); + if (!serialized_tensors.ContainsKey(Trackable.None)) + { + serialized_tensors[Trackable.None] = new Dictionary>>(); + } + serialized_tensors[Trackable.None][Trackable.Constants.OBJECT_GRAPH_PROTO_KEY] = object_graph_tensor; + return (serialized_tensors, feed_additions, registered_savers, graph_proto); + } + + private (Tensor, IDictionary) save_cached_when_graph_building(Tensor file_prefix, Tensor object_graph_tensor, CheckpointOptions options) + { + var (serialized_tensors, feed_additions, registered_savers, graph_proto) = gather_serialized_tensors(object_graph_tensor); + + Func<(Tensor, IDictionary)> run_save = () => + { + if (_last_save_object_graph != graph_proto || tf.Context.executing_eagerly() || ops.inside_function()) + { + var saver = new MultiDeviceSaver(serialized_tensors, registered_savers); + var save_op = saver.save(file_prefix, options); + + // tensorflow python: `with ops.device("/cpu:0"):` + using (ops.control_dependencies(new object[] { save_op })) + { + _cached_save_operation = array_ops.identity(file_prefix); + } + _last_save_object_graph = graph_proto; + } + return (_cached_save_operation, feed_additions); + }; + + if (options.experimental_enable_async_checkpoint) + { + throw new NotImplementedException(); + } + + return run_save(); + } + + private (Tensor, IDictionary) save_cached_when_graph_building(string file_prefix, Tensor object_graph_tensor, CheckpointOptions options) + { + var (serialized_tensors, feed_additions, registered_savers, graph_proto) = gather_serialized_tensors(object_graph_tensor); + + Func<(Tensor, IDictionary)> run_save = () => + { + if (_last_save_object_graph != graph_proto || tf.Context.executing_eagerly() || ops.inside_function()) + { + var saver = new MultiDeviceSaver(serialized_tensors, registered_savers); + var save_op = saver.save(file_prefix, options); + + // tensorflow python: `with ops.device("/cpu:0"):` + using (ops.control_dependencies(new object[] {save_op} )) + { + _cached_save_operation = array_ops.identity(tf.constant(file_prefix)); + } + _last_save_object_graph = graph_proto; + } + return (_cached_save_operation, feed_additions); + }; + + if (options.experimental_enable_async_checkpoint) + { + throw new NotImplementedException(); + } + + return run_save(); + } + + // TODO: parameter write_done_callback + public Tensor save(string file_prefix, int? checkpoint_number = null, Session? session = null, + CheckpointOptions? options = null) + { + if (options is null) + { + options = new CheckpointOptions(); + } + + Dictionary feed_dict = new(); + bool use_session = (!tf.Context.executing_eagerly() && !ops.inside_function()); + if (checkpoint_number is not null) + { + file_prefix = $"{file_prefix}-{checkpoint_number?.ToString()}"; + } + + Tensor file_prefix_tensor; + Tensor object_graph_tensor; + string file_prefix_to_save; + if (use_session) + { + if (_object_graph_feed_tensor is null) + { + // In python there is `with ops.device("/cpu:0")`. + _object_graph_feed_tensor = constant_op.constant("", TF_DataType.TF_STRING); + _file_prefix_feed_tensor = constant_op.constant("", TF_DataType.TF_STRING); + } + + object_graph_tensor = _object_graph_feed_tensor; + file_prefix_tensor = _file_prefix_feed_tensor; + feed_dict[file_prefix_tensor] = file_prefix; + file_prefix_to_save = ""; + } + else + { + // In python there is `with ops.device("/cpu:0")`. + file_prefix_tensor = ops.convert_to_tensor(file_prefix, TF_DataType.TF_STRING); + object_graph_tensor = null; + file_prefix_to_save = file_prefix; + } + + var (save_path, new_feed_additions) = + save_cached_when_graph_building(file_prefix_to_save, object_graph_tensor, options); + + if (new_feed_additions is not null) + { + foreach (var pair in new_feed_additions) + { + feed_dict.Add(pair.Key, pair.Value); + } + } + if(!use_session) + { + session = null; + } + else if (session is null) + { + session = new Session(); // In python it uses `get_session`. + } + + if (session is not null) + { + var s = feed_dict.Select(x => new FeedItem(x.Key, x.Value)).ToArray(); + return session.run((Tensor)save_path, s); + } + else if (use_session) + { + throw new RuntimeError($"Unable to save checkpoint to \"{file_prefix}\" " + + "in graph mode without a default session. Please use " + + "`with tf.Session():` to create a session."); + } + else + { + return save_path; + } + } +} \ No newline at end of file diff --git a/src/TensorFlowNET.Core/Checkpoint/functional_saver.cs b/src/TensorFlowNET.Core/Checkpoint/functional_saver.cs new file mode 100644 index 000000000..09904d684 --- /dev/null +++ b/src/TensorFlowNET.Core/Checkpoint/functional_saver.cs @@ -0,0 +1,540 @@ +using System; +using System.Buffers.Text; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Train; +using static Tensorflow.ApiDef.Types; +using static Tensorflow.CostGraphDef.Types; +using static Tensorflow.OptimizerOptions.Types; +using static Tensorflow.Binding; +using System.Text.RegularExpressions; +using System.Linq; +using Tensorflow.Operations; +using Tensorflow.Training; +using Tensorflow.Graphs; +using System.Xml.Linq; +using System.Diagnostics; +using RestoreFunc = System.Func; + +namespace Tensorflow.Checkpoint +{ + public class Maybe + { + private TA? _valueA = default(TA); + private TB? _valueB = default(TB); + private Type _type; + private bool _assignedTA; + public Maybe(TA value) + { + _valueA = value; + _type= typeof(TA); + _assignedTA = true; + } + public Maybe(TB value) + { + _valueB = value; + _type = typeof(TB); + _assignedTA = false; + } + + public Type DataType => _type; + + /// + /// Try to get the type T member of this instance. It returns true when TA or TB derive from T and is correspondingly assigned. + /// It returns + /// + /// + /// + /// + public bool TryGet(out T? res) + { + if(_valueA is T && _valueB is not T) + { + res = (T)(object)_valueA; + return _assignedTA; + } + else if(_valueA is not T && _valueB is T) + { + res = (T)(object)_valueB; + return !_assignedTA; + } + res = default(T); + return false; + } + + public bool IsTypeOrDeriveFrom() + { + if (_valueA is T && _valueB is not T) + { + return _assignedTA; + } + else if (_valueA is not T && _valueB is T) + { + return !_assignedTA; + } + else if (_valueA is T && _valueB is T) + { + return true; + } + else + { + return false; + } + } + + public T GetValue() + { + if (_valueA is T && _valueB is not T) + { + return (T)(object)_valueA; + } + else if (_valueA is not T && _valueB is T) + { + return (T)(object)_valueB; + } + else if (_valueA is T && _valueB is T) + { + throw new TypeError("The type is vague, this is always because TA and TB both derive from T."); + } + else + { + throw new TypeError($"Expected {typeof(TA)} or {typeof(TB)}, but got typeof{typeof(T)}."); + } + } + + public static implicit operator Maybe(TA a) + { + return new Maybe(a); + } + public static implicit operator Maybe(TB b) + { + return new Maybe(b); + } + } + internal class SingleDeviceSaver + { + private IDictionary>> _tensor_slice_dict; + public SingleDeviceSaver(IDictionary>> tensor_slice_dict) + { + _tensor_slice_dict = tensor_slice_dict; + } + public SingleDeviceSaver(IDictionary> tensor_slice_dict) + { + _tensor_slice_dict = tensor_slice_dict.ToDictionary( + x => x.Key, x => x.Value.ToDictionary( + y => y.Key, y => new Maybe(y.Value)) + as IDictionary>); + } + public SingleDeviceSaver(IDictionary> tensor_slice_dict) + { + _tensor_slice_dict = tensor_slice_dict.ToDictionary( + x => x.Key, x => x.Value.ToDictionary( + y => y.Key, y => new Maybe(y.Value)) + as IDictionary>); + } + public Operation? save(Tensor file_prefix, CheckpointOptions? options = null) + { + if(options is null) + { + options = new CheckpointOptions(); + } + List tensor_names = new(); + List tensors = new(); + List slice_specs = new(); + foreach(var pair in _tensor_slice_dict) + { + var checkpoint_key = pair.Key; + var tensor_slices = pair.Value; + foreach(var slice in tensor_slices) + { + var slice_spec = slice.Key; + var maybe_tensor = slice.Value; + if(maybe_tensor.TryGet(out var spec)) + { + var tensor_value = spec.tensor; + if (tensor_value is not null) + { + tensor_names.Add(spec.name); + tensors.Add(tensor_value); + slice_specs.Add(spec.slice_spec); + } + } + else + { + var tensor = maybe_tensor.GetValue(); + tensor_names.Add(checkpoint_key); + tensors.Add(tensor); + slice_specs.Add(slice_spec); + } + } + } + // TODO: specify the device. + return tf.io.save_v2(file_prefix, tensor_names.ToArray(), slice_specs.ToArray(), tensors.ToArray()); + } + + public Operation? save(string file_prefix, CheckpointOptions? options = null) => save(tf.constant(file_prefix, TF_DataType.TF_STRING), options); + + public IDictionary> restore(Tensor file_prefix, CheckpointOptions? options = null) + { + if(options is null) + { + options = new CheckpointOptions(); + } + List tensor_names = new(); + List tensor_dtypes = new(); + List slice_specs = new(); + + foreach(var pair in _tensor_slice_dict) + { + var checkpoint_key = pair.Key; + var tensor_slices = pair.Value; + foreach(var slice in tensor_slices) + { + var slice_spec = slice.Key; + var maybe_tensor = slice.Value; + // TODO: deal with other types. Currently only `SaveSpec` is allowed. + if(maybe_tensor.TryGet(out var spec)) + { + tensor_dtypes.Add(spec.dtype); + slice_specs.Add(spec.slice_spec); + tensor_names.Add(spec.name); + } + else + { + var tensor = maybe_tensor.GetValue(); + tensor_dtypes.Add(tensor.dtype); + slice_specs.Add(slice_spec); + tensor_names.Add(checkpoint_key); + } + } + } + + string restore_device = string.IsNullOrEmpty(options.experimental_io_device) ? "cpu:0": options.experimental_io_device!; + + // tf python has code `with ops.device(restore_device):` here. + tf.device(restore_device); // may be risky. + var restored_tensors = tf.io.restore_v2(file_prefix, tensor_names.ToArray(), slice_specs.ToArray(), tensor_dtypes.ToArray()); + + Dictionary> restored_tensor_dict = new(); + int idx = 0; + foreach(var pair in _tensor_slice_dict) + { + var checkpoint_key = pair.Key; + var tensor_slices = pair.Value; + foreach(var slice_spec in tensor_slices.Keys) + { + var restored_tensor = restored_tensors[idx++]; + if (!restored_tensor_dict.ContainsKey(checkpoint_key)) + { + restored_tensor_dict[checkpoint_key] = new Dictionary(); + } + restored_tensor_dict[checkpoint_key][slice_spec] = restored_tensor; + } + } + return restored_tensor_dict; + } + + public IDictionary> restore(string file_prefix, CheckpointOptions? options = null) => restore(tf.constant(file_prefix)); + } + /// + /// Saves checkpoints directly from multiple devices. + /// Note that this is a low-level utility which stores Tensors in the keys + /// specified by `SaveableObject`s.Higher-level utilities for object-based + /// checkpointing are built on top of it. + /// + public class MultiDeviceSaver + { + private Dictionary _single_device_savers; + private IDictionary _registered_savers; + private Dictionary<(string, string), RestoreFunc> _keys_to_restore_fn; + private Dictionary> _restore_fn_to_keys; + /// + /// + /// + /// A dictionary mapping `Trackable` to a tensor dict, which maps checkpoint_key -> (slice_spec ->) -> Tensor/SaveSpec. + /// + /// + public MultiDeviceSaver(IDictionary>>> serialized_tensors, + IDictionary>? registered_savers = null, bool call_with_mapped_capture = false) + { + _keys_to_restore_fn = new Dictionary<(string, string), RestoreFunc>(); + _restore_fn_to_keys = new Dictionary>(); + Dictionary>> tensors_by_device= new(); + + foreach(var pair in serialized_tensors) + { + var obj = pair.Key; + var tensor_dict = pair.Value; + RestoreFunc restore_fn; + if(obj == Trackable.None) + { + restore_fn = new RestoreFunc(x => null); + } + else + { + restore_fn = new RestoreFunc(x => + { + if(x is IDictionary>>) + { + return obj._restore_from_tensors(x as IDictionary>>); + } + throw new TypeError($"Expected `IDictionary>>` as input, got{x.GetType()}."); + }); + } + + foreach(var item in tensor_dict) + { + var checkpoint_key = item.Key; + IDictionary spec_to_tensor; + if(item.Value.TryGet(out var t)) + { + spec_to_tensor = new Dictionary(); + spec_to_tensor[""] = t; + } + else + { + spec_to_tensor = item.Value.GetValue>(); + } + + foreach(var spec in spec_to_tensor) + { + var slice_spec = spec.Key; + var tensor = spec.Value; + if(_keys_to_restore_fn.ContainsKey((checkpoint_key, slice_spec))) + { + throw new ValueError("Recieved multiple tensors with the same checkpoint key and " + + $"slice spec. This is invalid because one will overwrite the " + + $"other in the checkpoint. This indicates a bug in the Checkpoint key-generation."); + } + _keys_to_restore_fn[(checkpoint_key, slice_spec)] = restore_fn; + _restore_fn_to_keys.SetDefault(restore_fn, new List<(string, string)>()).Add((checkpoint_key, slice_spec)); + + // skip the process of device name because lack of API. + var host_device = tensor.Device; + var internal_dict = tensors_by_device.SetDefault(host_device, new Dictionary>()); + if (!internal_dict.ContainsKey(checkpoint_key)) + { + internal_dict[checkpoint_key] = new Dictionary(); + } + internal_dict[checkpoint_key][slice_spec] = tensor; + } + } + } + + _single_device_savers = tensors_by_device.ToDictionary(x => x.Key, x => new SingleDeviceSaver(x.Value)); + + _registered_savers = new Dictionary(); + if(registered_savers is not null && registered_savers.Count > 0) + { + // TODO: complete the implementation. + throw new NotImplementedException(); + } + } + + public Operation save(Tensor file_prefix, CheckpointOptions? options= null) + { + if(options is null) + { + options = new CheckpointOptions(); + } + + tf.device("CPU"); // may be risky. + var sharded_suffix = array_ops.where(gen_ops.regex_full_match(file_prefix, tf.constant(@"^s3://.*")), + constant_op.constant(".part"), constant_op.constant("_temp/part")); + var tmp_checkpoint_prefix = gen_ops.string_join(new Tensor[] { file_prefix, sharded_suffix }); + IDictionary registered_paths = _registered_savers.Keys.ToDictionary(x => x, x => registered_saver_filename(file_prefix, x)); + + Operation save_fn() + { + List saved_prefixes= new(); + foreach(var saver in _registered_savers) + { + // TODO: implementi it later. + throw new NotImplementedException(); + } + + int num_shards = _single_device_savers.Count; + List sharded_saves = new(); + var num_shards_tensor = constant_op.constant(num_shards, name: "num_shards"); + string? last_device = null; + int shard = 0; + foreach(var pair in _single_device_savers.OrderBy(x => x.Key)) + { + var device = pair.Key; + var saver = pair.Value; + last_device = device; + // skip the extra process of device name because of lack of API. + tf.device(device); + var shard_prefix = sharded_filename(tmp_checkpoint_prefix, shard, num_shards_tensor); + saved_prefixes.Add(shard_prefix); + sharded_saves.Add(saver.save(shard_prefix, options)); + } + using (var controller = ops.control_dependencies(sharded_saves.ToArray())) + { + string merge_device = string.IsNullOrEmpty(options.experimental_io_device) ? last_device : options.experimental_io_device; + tf.device(merge_device); + return gen_ops.merge_v2_checkpoints(saved_prefixes.ToArray(), tf.constant(file_prefix), delete_old_dirs: true); + } + } + + if(tf.Context.executing_eagerly() && _single_device_savers.Count > 1) + { + // TODO: implement it. Currently `autograph` does not support the function with non parameter. + throw new NotImplementedException(); + } + else + { + return save_fn(); + } + } + + public Operation save(string file_prefix, CheckpointOptions? options = null) => save(tf.constant(file_prefix), options); + + public IDictionary restore(Tensor file_prefix, CheckpointOptions? options = null) + { + if(options is null) + { + options = new CheckpointOptions(); + } + + IDictionary restore_func() + { + Dictionary>>> restore_fn_inputs = new(); + Dictionary restore_fn_input_count = _restore_fn_to_keys.ToDictionary(x => x.Key, x => x.Value.Count); + Dictionary restore_ops = new(); + + foreach(var single_saver in _single_device_savers.OrderBy(x => x.Key)) + { + var device = single_saver.Key; + var saver = single_saver.Value; + tf.device(device); + var restored_tensor_dict = saver.restore(file_prefix, options); + + foreach(var pair in restored_tensor_dict) + { + var checkpoint_key = pair.Key; + var slice_and_tensor = pair.Value; + foreach(var item in slice_and_tensor) + { + var slice_spec = item.Key; + var tensor = item.Value; + var restore_fn = _keys_to_restore_fn[(checkpoint_key, slice_spec)]; + var internal_dict = restore_fn_inputs.SetDefault(restore_fn, new Dictionary>>()); + if (!string.IsNullOrEmpty(slice_spec)) + { + if (!internal_dict.ContainsKey(checkpoint_key)) + { + Dictionary dict = new(); + dict[slice_spec] = tensor; + internal_dict[checkpoint_key] = new Maybe>(dict); + } + else + { + internal_dict[checkpoint_key].GetValue>()[slice_spec] = tensor; + } + } + else + { + internal_dict[checkpoint_key] = new Maybe>(tensor); + } + restore_fn_input_count[restore_fn]--; + + if (restore_fn_input_count[restore_fn] == 0) + { + Dictionary>> restored_tensors = new(); + foreach(var input in restore_fn_inputs[restore_fn]) + { + restored_tensors[TrackableUtils.extract_local_name(input.Key)] = input.Value; + } + var ret = restore_fn.DynamicInvoke(restored_tensors); + if(ret is IDictionary) + { + var dict = (IDictionary)ret; + restore_ops = restore_ops.Concat(dict).ToDictionary(x => x.Key, x => x.Value); + } + } + } + } + } + + foreach(var item in _registered_savers) + { + throw new NotImplementedException(); + } + return restore_ops; + } + + // TODO: complete the implementation. Currently skip it because of lack of API. + bool has_custom_device_saver = false; + + if (tf.Context.executing_eagerly() && (_single_device_savers.Count > 1 || has_custom_device_saver)) + { + // TODO: implement it. Currently `autograph` does not support the function with non parameter. + throw new NotImplementedException(); + } + else + { + return restore_func(); + } + } + + /// + /// Serializes to a SaverDef referencing the current graph. + /// + public SaverDef to_proto() + { + var filename_tensor = array_ops.placeholder(TF_DataType.TF_STRING, new int[] { }, "saver_filename"); + var traced_save_func = tf.autograph.to_graph(_traced_save, TF_DataType.TF_STRING); + var traced_restore_func = tf.autograph.to_graph(_traced_restore, TF_DataType.TF_STRING); + var save_tensor = traced_save_func(filename_tensor); + var restore_op = traced_restore_func(filename_tensor).op; + return new SaverDef() + { + FilenameTensorName = filename_tensor.name, + SaveTensorName = save_tensor.name, + RestoreOpName = restore_op.name, + Version = SaverDef.Types.CheckpointFormatVersion.V2 + }; + } + + private Tensor _traced_save(Tensor file_prefix) + { + var save_op = save(file_prefix); + tf.device("cpu:0"); + using (ops.control_dependencies(new object[]{ save_op })) + { + return array_ops.identity(file_prefix); + } + } + + private Tensor _traced_restore(Tensor file_prefix) + { + var restore_op = restore(file_prefix); + tf.device("cpu:0"); + using (ops.control_dependencies(restore_op.Values.ToArray())) + { + return array_ops.identity(file_prefix); + } + } + + public static MultiDeviceSaver from_saveables(IEnumerable saveables, IDictionary>? registered_savers = null, bool call_with_mapped_captures = false) + { + Dictionary>>> serialized_tensors = new(); + foreach (var saveable in saveables) + { + var trackable = new SaveableCompatibilityConverter(saveable, new List() { saveable }); + serialized_tensors[trackable] = trackable.serialize_to_tensors(); + } + return new MultiDeviceSaver(serialized_tensors, registered_savers, call_with_mapped_captures); + } + + private static Tensor registered_saver_filename(Tensor filename_tensor, string saver_name) + { + return gen_ops.string_join(new Tensor[] { filename_tensor, constant_op.constant($"-{saver_name}") }); + } + private static Tensor sharded_filename(Tensor filename_tensor, int shard, Tensor num_shards) + { + return gen_ops.sharded_filename(filename_tensor, tf.constant(shard), num_shards); + } + } +} diff --git a/src/TensorFlowNET.Core/DisposableObject.cs b/src/TensorFlowNET.Core/DisposableObject.cs index 3c70739bd..c3c677fff 100644 --- a/src/TensorFlowNET.Core/DisposableObject.cs +++ b/src/TensorFlowNET.Core/DisposableObject.cs @@ -17,6 +17,7 @@ limitations under the License. using System; using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; +using Tensorflow.Train; namespace Tensorflow { @@ -90,4 +91,71 @@ public void Dispose() Dispose(false); } } -} \ No newline at end of file + + public abstract class DisposableTrackableObject: Trackable, IDisposable + { + protected IntPtr _handle; + protected bool _disposed; + + protected DisposableTrackableObject() + { } + + protected DisposableTrackableObject(IntPtr handle) + => _handle = handle; + + private void Dispose(bool disposing) + { + if (_disposed) + return; + + //first handle managed, they might use the unmanaged resources. + if (disposing) + { + // dispose managed state (managed objects). + DisposeManagedResources(); + } + + // free unmanaged memory + if (_handle != IntPtr.Zero) + { + // Call the appropriate methods to clean up + // unmanaged resources here. + // If disposing is false, + // only the following code is executed. + DisposeUnmanagedResources(_handle); + _handle = IntPtr.Zero; + } + + // Note disposing has been done. + _disposed = true; + } + + /// + /// Dispose any managed resources. + /// + /// Equivalent to what you would perform inside + protected virtual void DisposeManagedResources() + { } + + /// + /// Dispose any unmanaged resources related to given . + /// + protected abstract void DisposeUnmanagedResources(IntPtr handle); + + public void Dispose() + { + Dispose(true); + // This object will be cleaned up by the Dispose method. + // Therefore, you should call GC.SupressFinalize to + // take this object off the finalization queue + // and prevent finalization code for this object + // from executing a second time. + GC.SuppressFinalize(this); + } + + ~DisposableTrackableObject() + { + Dispose(false); + } + } +} diff --git a/src/TensorFlowNET.Core/Eager/execute.cs b/src/TensorFlowNET.Core/Eager/execute.cs new file mode 100644 index 000000000..cb3ea4d3c --- /dev/null +++ b/src/TensorFlowNET.Core/Eager/execute.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Xml.Linq; +using Tensorflow.Contexts; +using static Tensorflow.ApiDef.Types; +using static Tensorflow.CostGraphDef.Types; +using static Tensorflow.Binding; + +namespace Tensorflow.Eager +{ + internal class execute + { + public static (DataType[], Tensor[]) onvert_to_mixed_eager_tensors(Tensor[] values, Context ctx) + { + var v = values.Select(t => ops.convert_to_tensor(t, ctx:ctx)); + var types = v.Select(t => t.dtype.as_datatype_enum()); + return (types.ToArray(), v.ToArray()); + } + public static Tensor[] quick_execute(string op_name, int num_outputs, Tensor[] inputs, object[] attrs, Context ctx, string name = null) + { + string device_name = ctx.DeviceName; + + ctx.ensure_initialized(); + var tensors = tf.Runner.TFE_Execute(ctx, device_name, op_name, inputs, attrs, num_outputs); + + return tensors; + } + } +} diff --git a/src/TensorFlowNET.Core/Exceptions/AssertionError.cs b/src/TensorFlowNET.Core/Exceptions/AssertionError.cs new file mode 100644 index 000000000..977fe2340 --- /dev/null +++ b/src/TensorFlowNET.Core/Exceptions/AssertionError.cs @@ -0,0 +1,14 @@ +namespace Tensorflow.Exceptions; + +public class AssertionError : TensorflowException +{ + public AssertionError() : base() + { + + } + + public AssertionError(string message) : base(message) + { + + } +} diff --git a/src/TensorFlowNET.Core/Framework/meta_graph.cs b/src/TensorFlowNET.Core/Framework/meta_graph.cs index 6ce3bf3c5..c3616fafd 100644 --- a/src/TensorFlowNET.Core/Framework/meta_graph.cs +++ b/src/TensorFlowNET.Core/Framework/meta_graph.cs @@ -304,7 +304,7 @@ private static void add_collection_def(MetaGraphDef meta_graph_def, } } - private static OpList stripped_op_list_for_graph(GraphDef graph_def) + public static OpList stripped_op_list_for_graph(GraphDef graph_def) { var used_ops = ops_used_by_graph_def(graph_def); @@ -345,5 +345,89 @@ private static string[] ops_used_by_graph_def(GraphDef graph_def) return used_ops.ToArray(); } + + private static bool is_default_attr_value(OpDef op_def, string attr_name, AttrValue attr_value) + { + foreach (var attr_def in op_def.Attr) + { + if (attr_def.Name == attr_name) + { + if (attr_def.DefaultValue is null) return false; + // TODO: add new c_api `EqualAttrValueWrapper` and complete the check. + return true; + } + } + + return false; + } + + public static void strip_graph_default_valued_attrs(MetaGraphDef meta_graph_def) + { + Dictionary op_name_to_function = new(); + foreach (var function_def in meta_graph_def.GraphDef.Library.Function) + { + op_name_to_function[function_def.Signature.Name] = function_def; + } + + Action _strip_node_default_valued_attrs = (node_def) => + { + if (op_name_to_function.ContainsKey(node_def.Op)) return; + + var op_def = op_def_registry.GetOpDef(node_def.Op); + if(op_def is null) return; + + HashSet attrs_to_strip = new(); + foreach (var attr in node_def.Attr) + { + if (is_default_attr_value(op_def, attr.Key, attr.Value)) + { + attrs_to_strip.Add(attr.Key); + } + } + + foreach (var attr in attrs_to_strip) + { + node_def.Attr.Remove(attr); + } + }; + + foreach (var node_def in meta_graph_def.GraphDef.Node) + { + _strip_node_default_valued_attrs(node_def); + } + + foreach (var function_def in meta_graph_def.GraphDef.Library.Function) + { + foreach (var function_node_def in function_def.NodeDef) + { + _strip_node_default_valued_attrs(function_node_def); + } + } + + meta_graph_def.MetaInfoDef.StrippedDefaultAttrs = true; + } + + /// + /// Extract the Op name from a Tensor name. + /// + /// + /// + public static string op_name(string tensor_name) + { + if (string.IsNullOrEmpty(tensor_name)) + { + throw new ValueError($"Tensor name cannot be empty or None. Received: {tensor_name}."); + } + + if (tensor_name.StartsWith("^")) + { + tensor_name = tensor_name.Substring(1); + } + if (tensor_name.Contains(":")) + { + return tensor_name.Split(':')[0]; + } + return tensor_name; + } } } diff --git a/src/TensorFlowNET.Core/Functions/ConcreteFunction.cs b/src/TensorFlowNET.Core/Functions/ConcreteFunction.cs index c52d0b5f5..bac9cedbf 100644 --- a/src/TensorFlowNET.Core/Functions/ConcreteFunction.cs +++ b/src/TensorFlowNET.Core/Functions/ConcreteFunction.cs @@ -3,6 +3,7 @@ using System.Linq; using Tensorflow.Framework.Models; using Tensorflow.Graphs; +using Tensorflow.Train; using static Tensorflow.Binding; namespace Tensorflow.Functions @@ -10,7 +11,7 @@ namespace Tensorflow.Functions /// /// /// - public class ConcreteFunction + public class ConcreteFunction: Trackable { FuncGraph func_graph; ForwardBackwardCall forward_backward; diff --git a/src/TensorFlowNET.Core/Functions/Function.cs b/src/TensorFlowNET.Core/Functions/Function.cs index d57097ae9..056d15f4d 100644 --- a/src/TensorFlowNET.Core/Functions/Function.cs +++ b/src/TensorFlowNET.Core/Functions/Function.cs @@ -1,16 +1,23 @@ using System; +using Tensorflow.Train; namespace Tensorflow { - public class Function + public class Function: Trackable { #pragma warning disable CS0169 // The field 'Function._handle' is never used private IntPtr _handle; #pragma warning restore CS0169 // The field 'Function._handle' is never used - + + public string Name { get; set; } public Function() { } + + public Function(string name) + { + Name = name; + } } } diff --git a/src/TensorFlowNET.Core/Graphs/AutoGraph.cs b/src/TensorFlowNET.Core/Graphs/AutoGraph.cs index 2af1a3720..ceeca8abf 100644 --- a/src/TensorFlowNET.Core/Graphs/AutoGraph.cs +++ b/src/TensorFlowNET.Core/Graphs/AutoGraph.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics; using System.Linq; using static Tensorflow.Binding; @@ -6,14 +7,14 @@ namespace Tensorflow.Graphs { public class AutoGraph { - public Func to_graph(Func func) + public Func to_graph(Func func, TF_DataType dtype = TF_DataType.TF_INT32) { string func_name = $"{func.Method.Name}_{ops.uid_function()}"; var graph = new FuncGraph(func_name); graph.as_default(); - var input = tf.placeholder(tf.int32); + var input = tf.placeholder(dtype); var output = func(input); var opers = graph._nodes_by_name.Values.Select(x => x as Operation).ToArray(); @@ -26,25 +27,33 @@ public Func to_graph(Func func) return (Tensor input) => { - var result = tf.Runner.TFE_Execute(tf.Context, - tf.Context.DeviceName, - func_name, - new[] { input }, - null, - 1); - return result[0]; + if (tf.executing_eagerly()) + { + var result = tf.Runner.TFE_Execute(tf.Context, + tf.Context.DeviceName, + func_name, + new[] { input }, + null, + 1); + return result[0]; + } + using (var s = tf.Session(input.graph)) + { + var output = func(input); + return output; + } }; } - public Func to_graph(Func func) + public Func to_graph(Func func, params TF_DataType[] dtypes) { string func_name = $"{func.Method.Name}_{ops.uid_function()}"; var graph = new FuncGraph(func_name); graph.as_default(); - var input1 = tf.placeholder(tf.int32); - var input2 = tf.placeholder(tf.int32); + var input1 = tf.placeholder(dtypes.Length >= 1 ? dtypes[0] : tf.int32); + var input2 = tf.placeholder(dtypes.Length >= 2 ? dtypes[1] : tf.int32); var output = func(input1, input2); var opers = graph._nodes_by_name.Values.Select(x => x as Operation).ToArray(); @@ -56,13 +65,22 @@ public Func to_graph(Func func) return (Tensor a, Tensor b) => { - var result = tf.Runner.TFE_Execute(tf.Context, + if (tf.executing_eagerly()) + { + var result = tf.Runner.TFE_Execute(tf.Context, tf.Context.DeviceName, func_name, new[] { a, b }, null, 1); - return result[0]; + return result[0]; + } + using (var s = tf.Session(a.graph)) + { + Debug.Assert(a.graph == b.graph); + var output = func(a, b); + return output; + } }; } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/ELUArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/ELUArgs.cs index 235523161..e830e5bf8 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/ELUArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/ELUArgs.cs @@ -1,9 +1,12 @@ -using System; +using Newtonsoft.Json; +using System; using System.Collections.Generic; using System.Text; namespace Tensorflow.Keras.ArgsDefinition { - public class ELUArgs : LayerArgs { - public float Alpha { get; set; } = 0.1f; - } + public class ELUArgs : AutoSerializeLayerArgs + { + [JsonProperty("alpha")] + public float Alpha { get; set; } = 0.1f; + } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/LeakyReLuArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/LeakyReLuArgs.cs index 6bdb294c2..6d9531346 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/LeakyReLuArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/LeakyReLuArgs.cs @@ -1,14 +1,16 @@ -using System; +using Newtonsoft.Json; +using System; using System.Collections.Generic; using System.Text; namespace Tensorflow.Keras.ArgsDefinition { - public class LeakyReLuArgs : LayerArgs + public class LeakyReLuArgs : AutoSerializeLayerArgs { /// /// Negative slope coefficient. /// + [JsonProperty("alpha")] public float Alpha { get; set; } = 0.3f; } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SoftmaxArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SoftmaxArgs.cs index ca35d75d5..1c1d147f1 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SoftmaxArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Activation/SoftmaxArgs.cs @@ -1,9 +1,12 @@ -using System; +using Newtonsoft.Json; +using System; using System.Collections.Generic; using System.Text; namespace Tensorflow.Keras.ArgsDefinition { - public class SoftmaxArgs : LayerArgs { - public Axis axis { get; set; } = -1; - } + public class SoftmaxArgs : AutoSerializeLayerArgs + { + [JsonProperty("axis")] + public Axis axis { get; set; } = -1; + } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Attention/AttentionArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Attention/AttentionArgs.cs index 73477c58f..4cdfb46bd 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Attention/AttentionArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Attention/AttentionArgs.cs @@ -1,3 +1,5 @@ +using Newtonsoft.Json; + namespace Tensorflow.Keras.ArgsDefinition { public class AttentionArgs : BaseDenseAttentionArgs @@ -6,6 +8,7 @@ public class AttentionArgs : BaseDenseAttentionArgs /// /// If `true`, will create a scalar variable to scale the attention scores. /// + [JsonProperty("use_scale")] public bool use_scale { get; set; } = false; /// @@ -14,6 +17,7 @@ public class AttentionArgs : BaseDenseAttentionArgs /// and key vectors. `"concat"` refers to the hyperbolic tangent of the /// concatenation of the query and key vectors. /// + [JsonProperty("score_mode")] public string score_mode { get; set; } = "dot"; } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Attention/BaseDenseAttentionArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Attention/BaseDenseAttentionArgs.cs index b2a0c3a51..0ef017370 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Attention/BaseDenseAttentionArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Attention/BaseDenseAttentionArgs.cs @@ -1,6 +1,8 @@ +using Newtonsoft.Json; + namespace Tensorflow.Keras.ArgsDefinition { - public class BaseDenseAttentionArgs : LayerArgs + public class BaseDenseAttentionArgs : AutoSerializeLayerArgs { /// @@ -14,6 +16,7 @@ public class BaseDenseAttentionArgs : LayerArgs /// Float between 0 and 1. Fraction of the units to drop for the /// attention scores. /// + [JsonProperty("dropout")] public float dropout { get; set; } = 0f; } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Attention/MultiHeadAttentionArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Attention/MultiHeadAttentionArgs.cs index 21b2d218c..077dea89d 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Attention/MultiHeadAttentionArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Attention/MultiHeadAttentionArgs.cs @@ -1,22 +1,40 @@ +using Newtonsoft.Json; using System; using static Tensorflow.Binding; namespace Tensorflow.Keras.ArgsDefinition { - public class MultiHeadAttentionArgs : LayerArgs + public class MultiHeadAttentionArgs : AutoSerializeLayerArgs { + [JsonProperty("num_heads")] public int NumHeads { get; set; } + [JsonProperty("key_dim")] public int KeyDim { get; set; } + [JsonProperty("value_dim")] public int? ValueDim { get; set; } = null; + [JsonProperty("dropout")] public float Dropout { get; set; } = 0f; + [JsonProperty("use_bias")] public bool UseBias { get; set; } = true; + [JsonProperty("output_shape")] public Shape OutputShape { get; set; } = null; + [JsonProperty("attention_axes")] public Shape AttentionAxis { get; set; } = null; + [JsonProperty("kernel_initializer")] public IInitializer KernelInitializer { get; set; } = tf.glorot_uniform_initializer; + [JsonProperty("bias_initializer")] public IInitializer BiasInitializer { get; set; } = tf.zeros_initializer; + [JsonProperty("kernel_regularizer")] public IRegularizer KernelRegularizer { get; set; } = null; + [JsonProperty("bias_regularizer")] public IRegularizer BiasRegularizer { get; set; } = null; + [JsonProperty("kernel_constraint")] public Action KernelConstraint { get; set; } = null; + [JsonProperty("bias_constraint")] public Action BiasConstraint { get; set; } = null; + [JsonProperty("activity_regularizer")] + public override IRegularizer ActivityRegularizer { get => base.ActivityRegularizer; set => base.ActivityRegularizer = value; } + + // TODO: Add `key_shape`, `value_shape`, `query_shape`. } } \ No newline at end of file diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/AutoSerializeLayerArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/AutoSerializeLayerArgs.cs new file mode 100644 index 000000000..1a97b0135 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/AutoSerializeLayerArgs.cs @@ -0,0 +1,25 @@ +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.ArgsDefinition +{ + /// + /// This class has nothing but the attributes different from `LayerArgs`. + /// It's used to serialize the model to `tf` format. + /// If the `get_config` of a `Layer` in python code of tensorflow contains `super().get_config`, + /// then the Arg definition should inherit `utoSerializeLayerArgs` instead of `LayerArgs`. + /// + public class AutoSerializeLayerArgs: LayerArgs + { + [JsonProperty("name")] + public override string Name { get => base.Name; set => base.Name = value; } + [JsonProperty("dtype")] + public override TF_DataType DType { get => base.DType; set => base.DType = value; } + [JsonProperty("batch_input_shape", NullValueHandling = NullValueHandling.Ignore)] + public override Shape BatchInputShape { get => base.BatchInputShape; set => base.BatchInputShape = value; } + [JsonProperty("trainable")] + public override bool Trainable { get => base.Trainable; set => base.Trainable = value; } + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Convolution/ConvolutionalArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Convolution/ConvolutionalArgs.cs index 4f050228b..08d563c1a 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Convolution/ConvolutionalArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Convolution/ConvolutionalArgs.cs @@ -1,31 +1,65 @@ -using System; +using Newtonsoft.Json; +using System; using static Tensorflow.Binding; namespace Tensorflow.Keras.ArgsDefinition { - public class ConvolutionalArgs : LayerArgs + public class ConvolutionalArgs : AutoSerializeLayerArgs { public int Rank { get; set; } = 2; + [JsonProperty("filters")] public int Filters { get; set; } public int NumSpatialDims { get; set; } = Unknown; + [JsonProperty("kernel_size")] public Shape KernelSize { get; set; } = 5; /// /// specifying the stride length of the convolution. /// + [JsonProperty("strides")] public Shape Strides { get; set; } = (1, 1); - + [JsonProperty("padding")] public string Padding { get; set; } = "valid"; + [JsonProperty("data_format")] public string DataFormat { get; set; } + [JsonProperty("dilation_rate")] public Shape DilationRate { get; set; } = (1, 1); + [JsonProperty("groups")] public int Groups { get; set; } = 1; public Activation Activation { get; set; } + private string _activationName; + [JsonProperty("activation")] + public string ActivationName + { + get + { + if (string.IsNullOrEmpty(_activationName)) + { + return Activation.Method.Name; + } + else + { + return _activationName; + } + } + set + { + _activationName = value; + } + } + [JsonProperty("use_bias")] public bool UseBias { get; set; } + [JsonProperty("kernel_initializer")] public IInitializer KernelInitializer { get; set; } = tf.glorot_uniform_initializer; + [JsonProperty("bias_initializer")] public IInitializer BiasInitializer { get; set; } = tf.zeros_initializer; + [JsonProperty("kernel_regularizer")] public IRegularizer KernelRegularizer { get; set; } + [JsonProperty("bias_regularizer")] public IRegularizer BiasRegularizer { get; set; } + [JsonProperty("kernel_constraint")] public Action KernelConstraint { get; set; } + [JsonProperty("bias_constraint")] public Action BiasConstraint { get; set; } } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Core/DenseArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Core/DenseArgs.cs index e9b3c2fd9..8f4facbd4 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Core/DenseArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Core/DenseArgs.cs @@ -1,13 +1,18 @@ -using System; +using Newtonsoft.Json; +using System; +using System.Xml.Linq; +using Tensorflow.Operations.Initializers; using static Tensorflow.Binding; namespace Tensorflow.Keras.ArgsDefinition { + // TODO: `activity_regularizer` public class DenseArgs : LayerArgs { /// /// Positive integer, dimensionality of the output space. /// + [JsonProperty("units")] public int Units { get; set; } /// @@ -15,39 +20,74 @@ public class DenseArgs : LayerArgs /// public Activation Activation { get; set; } + private string _activationName; + [JsonProperty("activation")] + public string ActivationName + { + get + { + if (string.IsNullOrEmpty(_activationName)) + { + return Activation.Method.Name; + } + else + { + return _activationName; + } + } + set + { + _activationName = value; + } + } + /// /// Whether the layer uses a bias vector. /// + [JsonProperty("use_bias")] public bool UseBias { get; set; } = true; /// /// Initializer for the `kernel` weights matrix. /// + [JsonProperty("kernel_initializer")] public IInitializer KernelInitializer { get; set; } = tf.glorot_uniform_initializer; /// /// Initializer for the bias vector. /// + [JsonProperty("bias_initializer")] public IInitializer BiasInitializer { get; set; } = tf.zeros_initializer; /// /// Regularizer function applied to the `kernel` weights matrix. /// + [JsonProperty("kernel_regularizer")] public IRegularizer KernelRegularizer { get; set; } /// /// Regularizer function applied to the bias vector. /// + [JsonProperty("bias_regularizer")] public IRegularizer BiasRegularizer { get; set; } /// /// Constraint function applied to the `kernel` weights matrix. /// + [JsonProperty("kernel_constraint")] public Action KernelConstraint { get; set; } /// /// Constraint function applied to the bias vector. /// + [JsonProperty("bias_constraint")] public Action BiasConstraint { get; set; } + + [JsonProperty("name")] + public override string Name { get => base.Name; set => base.Name = value; } + [JsonProperty("dtype")] + public override TF_DataType DType { get => base.DType; set => base.DType = value; } + [JsonProperty("trainable")] + public override bool Trainable { get => base.Trainable; set => base.Trainable = value; } } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Attention/EinsumDenseArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Core/EinsumDenseArgs.cs similarity index 65% rename from src/TensorFlowNET.Core/Keras/ArgsDefinition/Attention/EinsumDenseArgs.cs rename to src/TensorFlowNET.Core/Keras/ArgsDefinition/Core/EinsumDenseArgs.cs index 3a8642ffc..9817e9c6d 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Attention/EinsumDenseArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Core/EinsumDenseArgs.cs @@ -1,9 +1,10 @@ +using Newtonsoft.Json; using System; using static Tensorflow.Binding; -namespace Tensorflow.Keras.ArgsDefinition +namespace Tensorflow.Keras.ArgsDefinition.Core { - public class EinsumDenseArgs : LayerArgs + public class EinsumDenseArgs : AutoSerializeLayerArgs { /// /// An equation describing the einsum to perform. This equation must @@ -11,6 +12,7 @@ public class EinsumDenseArgs : LayerArgs /// `ab...,bc->ac...` where 'ab', 'bc', and 'ac' can be any valid einsum axis /// expression sequence. /// + [JsonProperty("equation")] public string Equation { get; set; } /// @@ -19,6 +21,7 @@ public class EinsumDenseArgs : LayerArgs /// None for any dimension that is unknown or can be inferred from the input /// shape. /// + [JsonProperty("output_shape")] public Shape OutputShape { get; set; } /// @@ -26,41 +29,70 @@ public class EinsumDenseArgs : LayerArgs /// Each character in the `bias_axes` string should correspond to a character /// in the output portion of the `equation` string. /// + [JsonProperty("bias_axes")] public string BiasAxes { get; set; } = null; /// /// Activation function to use. /// public Activation Activation { get; set; } + private string _activationName; + [JsonProperty("activation")] + public string ActivationName + { + get + { + if (string.IsNullOrEmpty(_activationName)) + { + return Activation.Method.Name; + } + else + { + return _activationName; + } + } + set + { + _activationName = value; + } + } /// /// Initializer for the `kernel` weights matrix. /// + [JsonProperty("kernel_initializer")] public IInitializer KernelInitializer { get; set; } = tf.glorot_uniform_initializer; /// /// Initializer for the bias vector. /// + [JsonProperty("bias_initializer")] public IInitializer BiasInitializer { get; set; } = tf.zeros_initializer; /// /// Regularizer function applied to the `kernel` weights matrix. /// + [JsonProperty("kernel_regularizer")] public IRegularizer KernelRegularizer { get; set; } /// /// Regularizer function applied to the bias vector. /// + [JsonProperty("bias_regularizer")] public IRegularizer BiasRegularizer { get; set; } /// /// Constraint function applied to the `kernel` weights matrix. /// + [JsonProperty("kernel_constraint")] public Action KernelConstraint { get; set; } /// /// Constraint function applied to the bias vector. /// + [JsonProperty("bias_constraint")] public Action BiasConstraint { get; set; } + [JsonProperty("activity_regularizer")] + public override IRegularizer ActivityRegularizer { get => base.ActivityRegularizer; set => base.ActivityRegularizer = value; } } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Core/EmbeddingArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Core/EmbeddingArgs.cs index b1f4fddd3..c462961b3 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Core/EmbeddingArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Core/EmbeddingArgs.cs @@ -1,11 +1,22 @@ -namespace Tensorflow.Keras.ArgsDefinition +using Newtonsoft.Json; + +namespace Tensorflow.Keras.ArgsDefinition { - public class EmbeddingArgs : LayerArgs + public class EmbeddingArgs : AutoSerializeLayerArgs { + [JsonProperty("input_dim")] public int InputDim { get; set; } + [JsonProperty("output_dim")] public int OutputDim { get; set; } + [JsonProperty("mask_zero")] public bool MaskZero { get; set; } + [JsonProperty("input_length")] public int InputLength { get; set; } = -1; + [JsonProperty("embeddings_initializer")] public IInitializer EmbeddingsInitializer { get; set; } + [JsonProperty("activity_regularizer")] + public override IRegularizer ActivityRegularizer { get => base.ActivityRegularizer; set => base.ActivityRegularizer = value; } + + // TODO: `embeddings_regularizer`, `embeddings_constraint`. } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Core/InputLayerArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Core/InputLayerArgs.cs index 723109c27..be43e0a62 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Core/InputLayerArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Core/InputLayerArgs.cs @@ -1,9 +1,22 @@ -namespace Tensorflow.Keras.ArgsDefinition +using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; +using Tensorflow.Keras.Common; + +namespace Tensorflow.Keras.ArgsDefinition { public class InputLayerArgs : LayerArgs { + [JsonIgnore] public Tensor InputTensor { get; set; } - public bool Sparse { get; set; } + [JsonProperty("sparse")] + public virtual bool Sparse { get; set; } + [JsonProperty("ragged")] public bool Ragged { get; set; } + [JsonProperty("name")] + public override string Name { get => base.Name; set => base.Name = value; } + [JsonProperty("dtype")] + public override TF_DataType DType { get => base.DType; set => base.DType = value; } + [JsonProperty("batch_input_shape", NullValueHandling = NullValueHandling.Ignore)] + public override Shape BatchInputShape { get => base.BatchInputShape; set => base.BatchInputShape = value; } } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Cropping/Cropping2DArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Cropping/Cropping2DArgs.cs deleted file mode 100644 index 16705063e..000000000 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Cropping/Cropping2DArgs.cs +++ /dev/null @@ -1,16 +0,0 @@ -using Tensorflow.NumPy; - -namespace Tensorflow.Keras.ArgsDefinition { - public class Cropping2DArgs : LayerArgs { - /// - /// channel last: (b, h, w, c) - /// channels_first: (b, c, h, w) - /// - public enum DataFormat { channels_first = 0, channels_last = 1 } - /// - /// Accept: int[1][2], int[1][1], int[2][2] - /// - public NDArray cropping { get; set; } - public DataFormat data_format { get; set; } = DataFormat.channels_last; - } -} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Cropping/Cropping3DArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Cropping/Cropping3DArgs.cs deleted file mode 100644 index 9da2adc7f..000000000 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Cropping/Cropping3DArgs.cs +++ /dev/null @@ -1,16 +0,0 @@ -using Tensorflow.NumPy; - -namespace Tensorflow.Keras.ArgsDefinition { - public class Cropping3DArgs : LayerArgs { - /// - /// channel last: (b, h, w, c) - /// channels_first: (b, c, h, w) - /// - public enum DataFormat { channels_first = 0, channels_last = 1 } - /// - /// Accept: int[1][3], int[1][1], int[3][2] - /// - public NDArray cropping { get; set; } - public DataFormat data_format { get; set; } = DataFormat.channels_last; - } -} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Cropping/CroppingArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Cropping/CroppingArgs.cs deleted file mode 100644 index 9d23acd43..000000000 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Cropping/CroppingArgs.cs +++ /dev/null @@ -1,10 +0,0 @@ -using Tensorflow.NumPy; - -namespace Tensorflow.Keras.ArgsDefinition { - public class CroppingArgs : LayerArgs { - /// - /// Accept length 1 or 2 - /// - public NDArray cropping { get; set; } - } -} \ No newline at end of file diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/DataAdapterArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/DataAdapterArgs.cs index f3cca438f..8ce1ec655 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/DataAdapterArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/DataAdapterArgs.cs @@ -1,8 +1,9 @@ using Tensorflow.Keras.Engine; +using Tensorflow.Keras.Saving; namespace Tensorflow.Keras.ArgsDefinition { - public class DataAdapterArgs + public class DataAdapterArgs: IKerasConfig { public Tensor X { get; set; } public Tensor Y { get; set; } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/DataHandlerArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/DataHandlerArgs.cs index b6e6849bc..fd603a85e 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/DataHandlerArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/DataHandlerArgs.cs @@ -1,8 +1,9 @@ using Tensorflow.Keras.Engine; +using Tensorflow.Keras.Saving; namespace Tensorflow.Keras.ArgsDefinition { - public class DataHandlerArgs + public class DataHandlerArgs: IKerasConfig { public Tensor X { get; set; } public Tensor Y { get; set; } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/LayerArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/LayerArgs.cs index 4df4fb2b4..febf14176 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/LayerArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/LayerArgs.cs @@ -1,51 +1,54 @@ -namespace Tensorflow.Keras.ArgsDefinition +using Newtonsoft.Json; +using Tensorflow.Keras.Saving; + +namespace Tensorflow.Keras.ArgsDefinition { - public class LayerArgs + [JsonObject(MemberSerialization.OptIn)] + public class LayerArgs: IKerasConfig { /// /// Indicates whether the layer's weights are updated during training /// and whether the layer's updates are run during training. /// - public bool Trainable { get; set; } = true; - - public string Name { get; set; } + public virtual bool Trainable { get; set; } = true; + public virtual string Name { get; set; } /// /// Only applicable to input layers. /// - public TF_DataType DType { get; set; } = TF_DataType.TF_FLOAT; + public virtual TF_DataType DType { get; set; } = TF_DataType.TF_FLOAT; /// /// Whether the `call` method can be used to build a TF graph without issues. /// This attribute has no effect if the model is created using the Functional /// API. Instead, `model.dynamic` is determined based on the internal layers. /// - public bool Dynamic { get; set; } = false; + public virtual bool Dynamic { get; set; } = false; /// /// Only applicable to input layers. /// - public Shape InputShape { get; set; } + public virtual Shape InputShape { get; set; } /// /// Only applicable to input layers. /// - public Shape BatchInputShape { get; set; } + public virtual Shape BatchInputShape { get; set; } - public int BatchSize { get; set; } = -1; + public virtual int BatchSize { get; set; } = -1; /// /// Initial weight values. /// - public float[] Weights { get; set; } + public virtual float[] Weights { get; set; } /// /// Regularizer function applied to the output of the layer(its "activation"). /// - public IRegularizer ActivityRegularizer { get; set; } + public virtual IRegularizer ActivityRegularizer { get; set; } - public bool Autocast { get; set; } + public virtual bool Autocast { get; set; } - public bool IsFromConfig { get; set; } + public virtual bool IsFromConfig { get; set; } } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Lstm/LSTMCellArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Lstm/LSTMCellArgs.cs deleted file mode 100644 index fb0868dc5..000000000 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Lstm/LSTMCellArgs.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace Tensorflow.Keras.ArgsDefinition.Lstm -{ - public class LSTMCellArgs : LayerArgs - { - } -} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/MergeArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/MergeArgs.cs index 3e6791e3b..0140b3dd0 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/MergeArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Merging/MergeArgs.cs @@ -4,6 +4,7 @@ namespace Tensorflow.Keras.ArgsDefinition { + // TODO: complete the implementation public class MergeArgs : LayerArgs { public Tensors Inputs { get; set; } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/NodeArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/NodeArgs.cs index 0d9e26ac4..ad55ff612 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/NodeArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/NodeArgs.cs @@ -1,6 +1,8 @@ -namespace Tensorflow.Keras.ArgsDefinition +using Tensorflow.Keras.Saving; + +namespace Tensorflow.Keras.ArgsDefinition { - public class NodeArgs + public class NodeArgs: IKerasConfig { public ILayer[] InboundLayers { get; set; } public int[] NodeIndices { get; set; } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Normalization/BatchNormalizationArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Normalization/BatchNormalizationArgs.cs index 954ede574..6ee91e80b 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Normalization/BatchNormalizationArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Normalization/BatchNormalizationArgs.cs @@ -1,21 +1,37 @@ -using static Tensorflow.Binding; +using Newtonsoft.Json; +using static Tensorflow.Binding; namespace Tensorflow.Keras.ArgsDefinition { - public class BatchNormalizationArgs : LayerArgs + public class BatchNormalizationArgs : AutoSerializeLayerArgs { + [JsonProperty("axis")] public Shape Axis { get; set; } = -1; + [JsonProperty("momentum")] public float Momentum { get; set; } = 0.99f; + [JsonProperty("epsilon")] public float Epsilon { get; set; } = 1e-3f; + [JsonProperty("center")] public bool Center { get; set; } = true; + [JsonProperty("scale")] public bool Scale { get; set; } = true; + [JsonProperty("beta_initializer")] public IInitializer BetaInitializer { get; set; } = tf.zeros_initializer; + [JsonProperty("gamma_initializer")] public IInitializer GammaInitializer { get; set; } = tf.ones_initializer; + [JsonProperty("moving_mean_initializer")] public IInitializer MovingMeanInitializer { get; set; } = tf.zeros_initializer; + [JsonProperty("moving_variance_initializer")] public IInitializer MovingVarianceInitializer { get; set; } = tf.ones_initializer; + [JsonProperty("beta_regularizer")] public IRegularizer BetaRegularizer { get; set; } + [JsonProperty("gamma_regularizer")] public IRegularizer GammaRegularizer { get; set; } + // TODO: `beta_constraint` and `gamma_constraint`. + [JsonProperty("renorm")] public bool Renorm { get; set; } + // TODO: `renorm_clipping` and `virtual_batch_size`. + [JsonProperty("renorm_momentum")] public float RenormMomentum { get; set; } = 0.99f; } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Normalization/LayerNormalizationArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Normalization/LayerNormalizationArgs.cs index 13fd98b41..1ac661b37 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Normalization/LayerNormalizationArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Normalization/LayerNormalizationArgs.cs @@ -1,16 +1,27 @@ -using static Tensorflow.Binding; +using Newtonsoft.Json; +using static Tensorflow.Binding; namespace Tensorflow.Keras.ArgsDefinition { - public class LayerNormalizationArgs : LayerArgs + public class LayerNormalizationArgs : AutoSerializeLayerArgs { + [JsonProperty("axis")] public Axis Axis { get; set; } = -1; + [JsonProperty("epsilon")] public float Epsilon { get; set; } = 1e-3f; + [JsonProperty("center")] public bool Center { get; set; } = true; + [JsonProperty("scale")] public bool Scale { get; set; } = true; + [JsonProperty("beta_initializer")] public IInitializer BetaInitializer { get; set; } = tf.zeros_initializer; + [JsonProperty("gamma_initializer")] public IInitializer GammaInitializer { get; set; } = tf.ones_initializer; + [JsonProperty("beta_regularizer")] public IRegularizer BetaRegularizer { get; set; } + [JsonProperty("gamma_regularizer")] public IRegularizer GammaRegularizer { get; set; } + + // TODO: `beta_constraint` and `gamma_constraint`. } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/OptimizerV2Args.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/OptimizerV2Args.cs index e2a0e43c8..6256fd329 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/OptimizerV2Args.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/OptimizerV2Args.cs @@ -1,6 +1,8 @@ -namespace Tensorflow.Keras.ArgsDefinition +using Tensorflow.Keras.Saving; + +namespace Tensorflow.Keras.ArgsDefinition { - public class OptimizerV2Args + public class OptimizerV2Args: IKerasConfig { public string Name { get; set; } public float LearningRate { get; set; } = 0.001f; diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/Pooling1DArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/Pooling1DArgs.cs index 9742203d6..c5fdca675 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/Pooling1DArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/Pooling1DArgs.cs @@ -1,6 +1,8 @@ -namespace Tensorflow.Keras.ArgsDefinition +using Newtonsoft.Json; + +namespace Tensorflow.Keras.ArgsDefinition { - public class Pooling1DArgs : LayerArgs + public class Pooling1DArgs : AutoSerializeLayerArgs { /// /// The pooling function to apply, e.g. `tf.nn.max_pool2d`. @@ -10,11 +12,13 @@ public class Pooling1DArgs : LayerArgs /// /// specifying the size of the pooling window. /// + [JsonProperty("pool_size")] public int PoolSize { get; set; } /// /// specifying the strides of the pooling operation. /// + [JsonProperty("strides")] public int Strides { get { return _strides.HasValue ? _strides.Value : PoolSize; } set { _strides = value; } @@ -24,11 +28,13 @@ public int Strides { /// /// The padding method, either 'valid' or 'same'. /// + [JsonProperty("padding")] public string Padding { get; set; } = "valid"; /// /// one of `channels_last` (default) or `channels_first`. /// + [JsonProperty("data_format")] public string DataFormat { get; set; } } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/Pooling2DArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/Pooling2DArgs.cs index 1260af4c6..91a372ef3 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/Pooling2DArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Pooling/Pooling2DArgs.cs @@ -1,6 +1,8 @@ -namespace Tensorflow.Keras.ArgsDefinition +using Newtonsoft.Json; + +namespace Tensorflow.Keras.ArgsDefinition { - public class Pooling2DArgs : LayerArgs + public class Pooling2DArgs : AutoSerializeLayerArgs { /// /// The pooling function to apply, e.g. `tf.nn.max_pool2d`. @@ -10,21 +12,25 @@ public class Pooling2DArgs : LayerArgs /// /// specifying the size of the pooling window. /// + [JsonProperty("pool_size")] public Shape PoolSize { get; set; } /// /// specifying the strides of the pooling operation. /// + [JsonProperty("strides")] public Shape Strides { get; set; } /// /// The padding method, either 'valid' or 'same'. /// + [JsonProperty("padding")] public string Padding { get; set; } = "valid"; /// /// one of `channels_last` (default) or `channels_first`. /// + [JsonProperty("data_format")] public string DataFormat { get; set; } } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Preprocessing/PreprocessingLayerArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Preprocessing/PreprocessingLayerArgs.cs index 28ccf9f74..97cb364d9 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Preprocessing/PreprocessingLayerArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Preprocessing/PreprocessingLayerArgs.cs @@ -4,7 +4,7 @@ namespace Tensorflow.Keras.ArgsDefinition { - public class PreprocessingLayerArgs : LayerArgs + public class PreprocessingLayerArgs : AutoSerializeLayerArgs { } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Preprocessing/RescalingArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Preprocessing/RescalingArgs.cs new file mode 100644 index 000000000..154bd8c89 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Preprocessing/RescalingArgs.cs @@ -0,0 +1,12 @@ +using Newtonsoft.Json; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class RescalingArgs : AutoSerializeLayerArgs + { + [JsonProperty("scale")] + public float Scale { get; set; } + [JsonProperty("offset")] + public float Offset { get; set; } + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Preprocessing/ResizingArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Preprocessing/ResizingArgs.cs index cf11595e2..39fa52211 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Preprocessing/ResizingArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Preprocessing/ResizingArgs.cs @@ -1,5 +1,6 @@ namespace Tensorflow.Keras.ArgsDefinition { + // TODO: no corresponding class found in keras python, maybe obselete? public class ResizingArgs : PreprocessingLayerArgs { public int Height { get; set; } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Preprocessing/TextVectorizationArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Preprocessing/TextVectorizationArgs.cs index ddeadc001..1a7149f5a 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Preprocessing/TextVectorizationArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Preprocessing/TextVectorizationArgs.cs @@ -1,4 +1,5 @@ -using System; +using Newtonsoft.Json; +using System; using System.Collections.Generic; using System.Text; @@ -6,11 +7,19 @@ namespace Tensorflow.Keras.ArgsDefinition { public class TextVectorizationArgs : PreprocessingLayerArgs { + [JsonProperty("standardize")] public Func Standardize { get; set; } + [JsonProperty("split")] public string Split { get; set; } = "standardize"; + [JsonProperty("max_tokens")] public int MaxTokens { get; set; } = -1; + [JsonProperty("output_mode")] public string OutputMode { get; set; } = "int"; + [JsonProperty("output_sequence_length")] public int OutputSequenceLength { get; set; } = -1; + [JsonProperty("vocabulary")] public string[] Vocabulary { get; set; } + + // TODO: Add `ngrams`, `sparse`, `ragged`, `idf_weights`, `encoding` } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Regularization/DropoutArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Regularization/DropoutArgs.cs index c41c6fe85..1c85d4936 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Regularization/DropoutArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Regularization/DropoutArgs.cs @@ -1,21 +1,26 @@ -namespace Tensorflow.Keras.ArgsDefinition +using Newtonsoft.Json; + +namespace Tensorflow.Keras.ArgsDefinition { - public class DropoutArgs : LayerArgs + public class DropoutArgs : AutoSerializeLayerArgs { /// /// Float between 0 and 1. Fraction of the input units to drop. /// + [JsonProperty("rate")] public float Rate { get; set; } /// /// 1D integer tensor representing the shape of the /// binary dropout mask that will be multiplied with the input. /// + [JsonProperty("noise_shape")] public Shape NoiseShape { get; set; } /// /// random seed. /// + [JsonProperty("seed")] public int? Seed { get; set; } public bool SupportsMasking { get; set; } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rescaling/RescalingArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rescaling/RescalingArgs.cs deleted file mode 100644 index ec9b53150..000000000 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rescaling/RescalingArgs.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Tensorflow.Keras.ArgsDefinition -{ - public class RescalingArgs : LayerArgs - { - public float Scale { get; set; } - public float Offset { get; set; } - } -} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/Cropping2DArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/Cropping2DArgs.cs new file mode 100644 index 000000000..8c2626390 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/Cropping2DArgs.cs @@ -0,0 +1,18 @@ +using Tensorflow.NumPy; + +namespace Tensorflow.Keras.ArgsDefinition.Reshaping +{ + public class Cropping2DArgs : LayerArgs + { + /// + /// channel last: (b, h, w, c) + /// channels_first: (b, c, h, w) + /// + public enum DataFormat { channels_first = 0, channels_last = 1 } + /// + /// Accept: int[1][2], int[1][1], int[2][2] + /// + public NDArray cropping { get; set; } + public DataFormat data_format { get; set; } = DataFormat.channels_last; + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/Cropping3DArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/Cropping3DArgs.cs new file mode 100644 index 000000000..2d98e55db --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/Cropping3DArgs.cs @@ -0,0 +1,18 @@ +using Tensorflow.NumPy; + +namespace Tensorflow.Keras.ArgsDefinition.Reshaping +{ + public class Cropping3DArgs : LayerArgs + { + /// + /// channel last: (b, h, w, c) + /// channels_first: (b, c, h, w) + /// + public enum DataFormat { channels_first = 0, channels_last = 1 } + /// + /// Accept: int[1][3], int[1][1], int[3][2] + /// + public NDArray cropping { get; set; } + public DataFormat data_format { get; set; } = DataFormat.channels_last; + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/CroppingArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/CroppingArgs.cs new file mode 100644 index 000000000..21b85966b --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/CroppingArgs.cs @@ -0,0 +1,12 @@ +using Tensorflow.NumPy; + +namespace Tensorflow.Keras.ArgsDefinition.Reshaping +{ + public class Cropping1DArgs : LayerArgs + { + /// + /// Accept length 1 or 2 + /// + public NDArray cropping { get; set; } + } +} \ No newline at end of file diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/FlattenArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/FlattenArgs.cs index c2b48cc2f..91ffc2058 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/FlattenArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/FlattenArgs.cs @@ -1,7 +1,10 @@ -namespace Tensorflow.Keras.ArgsDefinition +using Newtonsoft.Json; + +namespace Tensorflow.Keras.ArgsDefinition { - public class FlattenArgs : LayerArgs + public class FlattenArgs : AutoSerializeLayerArgs { + [JsonProperty("data_format")] public string DataFormat { get; set; } } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/PermuteArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/PermuteArgs.cs index 2686f6cd7..92be10ab1 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/PermuteArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/PermuteArgs.cs @@ -1,5 +1,9 @@ -namespace Tensorflow.Keras.ArgsDefinition { - public class PermuteArgs : LayerArgs { - public int[] dims { get; set; } - } +using Newtonsoft.Json; + +namespace Tensorflow.Keras.ArgsDefinition { + public class PermuteArgs : AutoSerializeLayerArgs + { + [JsonProperty("dims")] + public int[] dims { get; set; } + } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/ReshapeArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/ReshapeArgs.cs index 77bca8ad0..4d1123c8a 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/ReshapeArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/ReshapeArgs.cs @@ -1,7 +1,10 @@ -namespace Tensorflow.Keras.ArgsDefinition +using Newtonsoft.Json; + +namespace Tensorflow.Keras.ArgsDefinition { - public class ReshapeArgs : LayerArgs + public class ReshapeArgs : AutoSerializeLayerArgs { + [JsonProperty("target_shape")] public Shape TargetShape { get; set; } public object[] TargetShapeObjects { get; set; } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/UpSampling2DArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/UpSampling2DArgs.cs index 7fdda32d3..b35e0e4b6 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/UpSampling2DArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/UpSampling2DArgs.cs @@ -1,12 +1,17 @@ -namespace Tensorflow.Keras.ArgsDefinition +using Newtonsoft.Json; + +namespace Tensorflow.Keras.ArgsDefinition { - public class UpSampling2DArgs : LayerArgs + public class UpSampling2DArgs : AutoSerializeLayerArgs { + [JsonProperty("size")] public Shape Size { get; set; } + [JsonProperty("data_format")] public string DataFormat { get; set; } /// /// 'nearest', 'bilinear' /// + [JsonProperty("interpolation")] public string Interpolation { get; set; } = "nearest"; } } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/ZeroPadding2DArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/ZeroPadding2DArgs.cs index ed6e7cc9c..4831e435b 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/ZeroPadding2DArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Reshaping/ZeroPadding2DArgs.cs @@ -2,6 +2,7 @@ namespace Tensorflow.Keras.ArgsDefinition { + // TODO: complete the implementation public class ZeroPadding2DArgs : LayerArgs { public NDArray Padding { get; set; } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Lstm/LSTMArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMArgs.cs similarity index 67% rename from src/TensorFlowNET.Core/Keras/ArgsDefinition/Lstm/LSTMArgs.cs rename to src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMArgs.cs index b08d21d88..764641474 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Lstm/LSTMArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMArgs.cs @@ -1,9 +1,8 @@ -using Tensorflow.Keras.ArgsDefinition.Rnn; - -namespace Tensorflow.Keras.ArgsDefinition.Lstm +namespace Tensorflow.Keras.ArgsDefinition.Rnn { public class LSTMArgs : RNNArgs { + // TODO: maybe change the `RNNArgs` and implement this class. public bool UnitForgetBias { get; set; } public float Dropout { get; set; } public float RecurrentDropout { get; set; } diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs new file mode 100644 index 000000000..594c99bb0 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/LSTMCellArgs.cs @@ -0,0 +1,7 @@ +namespace Tensorflow.Keras.ArgsDefinition.Rnn +{ + // TODO: complete the implementation + public class LSTMCellArgs : LayerArgs + { + } +} diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs index da5279257..2585592c1 100644 --- a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Rnn/RNNArgs.cs @@ -1,21 +1,30 @@ -using System.Collections.Generic; +using Newtonsoft.Json; +using System.Collections.Generic; namespace Tensorflow.Keras.ArgsDefinition.Rnn { - public class RNNArgs : LayerArgs + public class RNNArgs : AutoSerializeLayerArgs { public interface IRnnArgCell : ILayer { object state_size { get; } } - + [JsonProperty("cell")] + // TODO: the cell should be serialized with `serialize_keras_object`. public IRnnArgCell Cell { get; set; } = null; + [JsonProperty("return_sequences")] public bool ReturnSequences { get; set; } = false; + [JsonProperty("return_state")] public bool ReturnState { get; set; } = false; + [JsonProperty("go_backwards")] public bool GoBackwards { get; set; } = false; + [JsonProperty("stateful")] public bool Stateful { get; set; } = false; + [JsonProperty("unroll")] public bool Unroll { get; set; } = false; + [JsonProperty("time_major")] public bool TimeMajor { get; set; } = false; + // TODO: Add `num_constants` and `zero_output_for_mask`. public Dictionary Kwargs { get; set; } = null; public int Units { get; set; } diff --git a/src/TensorFlowNET.Core/Keras/Common/CustomizedActivationJsonConverter.cs b/src/TensorFlowNET.Core/Keras/Common/CustomizedActivationJsonConverter.cs new file mode 100644 index 000000000..1bc13caf3 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Common/CustomizedActivationJsonConverter.cs @@ -0,0 +1,50 @@ +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.Common +{ + public class CustomizedActivationJsonConverter : JsonConverter + { + public override bool CanConvert(Type objectType) + { + return objectType == typeof(Activation); + } + + public override bool CanRead => true; + + public override bool CanWrite => true; + + public override void WriteJson(JsonWriter writer, object? value, JsonSerializer serializer) + { + if (value is null) + { + var token = JToken.FromObject(""); + token.WriteTo(writer); + } + else if (value is not Activation) + { + throw new TypeError($"Unable to use `CustomizedActivationJsonConverter` to serialize the type {value.GetType()}."); + } + else + { + var token = JToken.FromObject((value as Activation)!.GetType().Name); + token.WriteTo(writer); + } + } + + public override object? ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer) + { + throw new NotImplementedException(); + //var dims = serializer.Deserialize(reader, typeof(string)); + //if (dims is null) + //{ + // throw new ValueError("Cannot deserialize 'null' to `Activation`."); + //} + //return new Shape((long[])(dims!)); + } + } +} diff --git a/src/TensorFlowNET.Core/Keras/Common/CustomizedAxisJsonConverter.cs b/src/TensorFlowNET.Core/Keras/Common/CustomizedAxisJsonConverter.cs new file mode 100644 index 000000000..4e190605c --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Common/CustomizedAxisJsonConverter.cs @@ -0,0 +1,48 @@ +using Newtonsoft.Json.Linq; +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.Common +{ + public class CustomizedAxisJsonConverter : JsonConverter + { + public override bool CanConvert(Type objectType) + { + return objectType == typeof(Axis); + } + + public override bool CanRead => true; + + public override bool CanWrite => true; + + public override void WriteJson(JsonWriter writer, object? value, JsonSerializer serializer) + { + if (value is null) + { + var token = JToken.FromObject(new int[] { }); + token.WriteTo(writer); + } + else if (value is not Axis) + { + throw new TypeError($"Unable to use `CustomizedAxisJsonConverter` to serialize the type {value.GetType()}."); + } + else + { + var token = JToken.FromObject((value as Axis)!.axis); + token.WriteTo(writer); + } + } + + public override object? ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer) + { + var axis = serializer.Deserialize(reader, typeof(long[])); + if (axis is null) + { + throw new ValueError("Cannot deserialize 'null' to `Axis`."); + } + return new Axis((int[])(axis!)); + } + } +} diff --git a/src/TensorFlowNET.Core/Keras/Common/CustomizedNodeConfigJsonConverter.cs b/src/TensorFlowNET.Core/Keras/Common/CustomizedNodeConfigJsonConverter.cs new file mode 100644 index 000000000..1ad19fc89 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Common/CustomizedNodeConfigJsonConverter.cs @@ -0,0 +1,73 @@ +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Tensorflow.Keras.Saving; + +namespace Tensorflow.Keras.Common +{ + public class CustomizedNodeConfigJsonConverter : JsonConverter + { + public override bool CanConvert(Type objectType) + { + return objectType == typeof(NodeConfig); + } + + public override bool CanRead => true; + + public override bool CanWrite => true; + + public override void WriteJson(JsonWriter writer, object? value, JsonSerializer serializer) + { + if (value is null) + { + var token = JToken.FromObject(null); + token.WriteTo(writer); + } + else if (value is not NodeConfig) + { + throw new TypeError($"Unable to use `CustomizedNodeConfigJsonConverter` to serialize the type {value.GetType()}."); + } + else + { + var config = value as NodeConfig; + var token = JToken.FromObject(new object[] { config!.Name, config.NodeIndex, config.TensorIndex }); + token.WriteTo(writer); + } + } + + public override object? ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer) + { + var values = serializer.Deserialize(reader, typeof(object[])) as object[]; + if (values is null) + { + throw new ValueError("Cannot deserialize 'null' to `Shape`."); + } + if(values.Length != 3) + { + throw new ValueError($"The value ({string.Join(", ", values)}) cannot be deserialized to type `NodeConfig`."); + } + if (values[0] is not string) + { + throw new TypeError($"The first value of `NodeConfig` is expected to be `string`, but got `{values[0].GetType().Name}`"); + } + if (values[1] is not int) + { + throw new TypeError($"The first value of `NodeConfig` is expected to be `int`, but got `{values[1].GetType().Name}`"); + } + if (values[2] is not int) + { + throw new TypeError($"The first value of `NodeConfig` is expected to be `int`, but got `{values[2].GetType().Name}`"); + } + return new NodeConfig() + { + Name = values[0] as string, + NodeIndex = (int)values[1], + TensorIndex = (int)values[2] + }; + } + } +} diff --git a/src/TensorFlowNET.Core/Keras/Common/CustomizedShapeJsonConverter.cs b/src/TensorFlowNET.Core/Keras/Common/CustomizedShapeJsonConverter.cs new file mode 100644 index 000000000..300cb2f28 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Common/CustomizedShapeJsonConverter.cs @@ -0,0 +1,67 @@ +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.Common +{ + public class CustomizedShapeJsonConverter: JsonConverter + { + public override bool CanConvert(Type objectType) + { + return objectType == typeof(Shape); + } + + public override bool CanRead => true; + + public override bool CanWrite => true; + + public override void WriteJson(JsonWriter writer, object? value, JsonSerializer serializer) + { + if(value is null) + { + var token = JToken.FromObject(null); + token.WriteTo(writer); + } + else if(value is not Shape) + { + throw new TypeError($"Unable to use `CustomizedShapeJsonConverter` to serialize the type {value.GetType()}."); + } + else + { + var shape = (value as Shape)!; + long?[] dims = new long?[shape.ndim]; + for(int i = 0; i < dims.Length; i++) + { + if (shape.dims[i] == -1) + { + dims[i] = null; + } + else + { + dims[i] = shape.dims[i]; + } + } + var token = JToken.FromObject(dims); + token.WriteTo(writer); + } + } + + public override object? ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer) + { + var dims = serializer.Deserialize(reader, typeof(long?[])) as long?[]; + if(dims is null) + { + throw new ValueError("Cannot deserialize 'null' to `Shape`."); + } + long[] convertedDims = new long[dims.Length]; + for(int i = 0; i < dims.Length; i++) + { + convertedDims[i] = dims[i] ?? (-1); + } + return new Shape(convertedDims); + } + } +} diff --git a/src/TensorFlowNET.Core/Keras/Engine/InputSpec.cs b/src/TensorFlowNET.Core/Keras/Engine/InputSpec.cs index 7280594b7..6743935c8 100644 --- a/src/TensorFlowNET.Core/Keras/Engine/InputSpec.cs +++ b/src/TensorFlowNET.Core/Keras/Engine/InputSpec.cs @@ -16,23 +16,27 @@ limitations under the License. using System.Collections.Generic; using System.Linq; +using Tensorflow.Keras.Saving; namespace Tensorflow.Keras.Engine { /// /// Specifies the ndim, dtype and shape of every input to a layer. /// - public class InputSpec + public class InputSpec: IKerasConfigable { public int? ndim; + public int? max_ndim; public int? min_ndim; Dictionary axes; Shape shape; + TF_DataType dtype; public int[] AllAxisDim; public InputSpec(TF_DataType dtype = TF_DataType.DtInvalid, int? ndim = null, int? min_ndim = null, + int? max_ndim = null, Dictionary axes = null, Shape shape = null) { @@ -41,7 +45,9 @@ public InputSpec(TF_DataType dtype = TF_DataType.DtInvalid, axes = new Dictionary(); this.axes = axes; this.min_ndim = min_ndim; + this.max_ndim = max_ndim; this.shape = shape; + this.dtype = dtype; if (ndim == null && shape != null) this.ndim = shape.ndim; @@ -49,7 +55,30 @@ public InputSpec(TF_DataType dtype = TF_DataType.DtInvalid, AllAxisDim = axes.Select(x => x.Value).ToArray(); } + public IKerasConfig get_config() + { + return new Config() + { + DType = dtype == TF_DataType.DtInvalid ? null : dtype, + Shape = shape, + Ndim = ndim, + MinNdim = min_ndim, + MaxNdim = max_ndim, + Axes = axes.ToDictionary(x => x.Key.ToString(), x => x.Value) + }; + } + public override string ToString() => $"ndim={ndim}, min_ndim={min_ndim}, axes={axes.Count}"; + + public class Config: IKerasConfig + { + public TF_DataType? DType { get; set; } + public Shape Shape { get; set; } + public int? Ndim { get; set; } + public int? MinNdim { get;set; } + public int? MaxNdim { get;set; } + public IDictionary Axes { get; set; } + } } } diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs index 1ec4a2c6e..036291076 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs @@ -1,10 +1,12 @@ using System.Collections.Generic; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; +using Tensorflow.Keras.Saving; +using Tensorflow.Training; namespace Tensorflow.Keras { - public interface ILayer + public interface ILayer: IWithTrackable, IKerasConfigable { string Name { get; } bool Trainable { get; } @@ -19,8 +21,8 @@ public interface ILayer List NonTrainableWeights { get; } Shape OutputShape { get; } Shape BatchInputShape { get; } + TensorShapeConfig BuildInputShape { get; } TF_DataType DType { get; } int count_params(); - LayerArgs get_config(); } } diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Cropping.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Cropping.cs index 602e7a880..3578652ee 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Cropping.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.Cropping.cs @@ -1,5 +1,5 @@ using System; -using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.ArgsDefinition.Reshaping; using Tensorflow.NumPy; namespace Tensorflow.Keras.Layers diff --git a/src/TensorFlowNET.Core/Keras/Saving/IKerasConfig.cs b/src/TensorFlowNET.Core/Keras/Saving/IKerasConfig.cs new file mode 100644 index 000000000..1217e1e52 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Saving/IKerasConfig.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow.Keras.Saving +{ + public interface IKerasConfig + { + } + + public interface IKerasConfigable + { + IKerasConfig get_config(); + } +} diff --git a/src/TensorFlowNET.Core/Keras/Saving/LayerConfig.cs b/src/TensorFlowNET.Core/Keras/Saving/LayerConfig.cs index b8b8cab40..4ce290c83 100644 --- a/src/TensorFlowNET.Core/Keras/Saving/LayerConfig.cs +++ b/src/TensorFlowNET.Core/Keras/Saving/LayerConfig.cs @@ -1,4 +1,5 @@ -using System; +using Newtonsoft.Json; +using System; using System.Collections.Generic; using System.Text; using Tensorflow.Keras.ArgsDefinition; @@ -6,11 +7,15 @@ namespace Tensorflow.Keras.Saving { - public class LayerConfig + public class LayerConfig: IKerasConfig { + [JsonProperty("name")] public string Name { get; set; } + [JsonProperty("class_name")] public string ClassName { get; set; } + [JsonProperty("config")] public LayerArgs Config { get; set; } + [JsonProperty("inbound_nodes")] public List InboundNodes { get; set; } } } diff --git a/src/TensorFlowNET.Core/Keras/Saving/ModelConfig.cs b/src/TensorFlowNET.Core/Keras/Saving/ModelConfig.cs index abfb235be..cac19180f 100644 --- a/src/TensorFlowNET.Core/Keras/Saving/ModelConfig.cs +++ b/src/TensorFlowNET.Core/Keras/Saving/ModelConfig.cs @@ -1,15 +1,20 @@ -using System; +using Newtonsoft.Json; +using System; using System.Collections.Generic; using System.Text; using Tensorflow.Keras.Engine; namespace Tensorflow.Keras.Saving { - public class ModelConfig + public class ModelConfig : IKerasConfig { + [JsonProperty("name")] public string Name { get; set; } + [JsonProperty("layers")] public List Layers { get; set; } + [JsonProperty("input_layers")] public List InputLayers { get; set; } + [JsonProperty("output_layers")] public List OutputLayers { get; set; } public override string ToString() diff --git a/src/TensorFlowNET.Core/Keras/Saving/NodeConfig.cs b/src/TensorFlowNET.Core/Keras/Saving/NodeConfig.cs index 3132248ef..20e2fef59 100644 --- a/src/TensorFlowNET.Core/Keras/Saving/NodeConfig.cs +++ b/src/TensorFlowNET.Core/Keras/Saving/NodeConfig.cs @@ -1,10 +1,13 @@ -using System; +using Newtonsoft.Json; +using System; using System.Collections.Generic; using System.Text; +using Tensorflow.Keras.Common; namespace Tensorflow.Keras.Saving { - public class NodeConfig + [JsonConverter(typeof(CustomizedNodeConfigJsonConverter))] + public class NodeConfig : IKerasConfig { public string Name { get; set; } public int NodeIndex { get; set; } diff --git a/src/TensorFlowNET.Core/Keras/Saving/SavedModel/ISerializedAttributes.cs b/src/TensorFlowNET.Core/Keras/Saving/SavedModel/ISerializedAttributes.cs new file mode 100644 index 000000000..ae8a1ab13 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Saving/SavedModel/ISerializedAttributes.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; +using Tensorflow.Train; + +namespace Tensorflow.Keras.Saving.SavedModel +{ + public interface ISerializedAttributes + { + IDictionary Functions { get; } + + IDictionary CheckpointableObjects { get; } + + /// + /// Returns functions to attach to the root object during serialization. + /// + IDictionary FunctionsToSerialize { get; } + + /// + /// Returns objects to attach to the root object during serialization. + /// + IDictionary ObjectsToSerialize{get; } + + /// + /// Saves function dictionary, and validates dictionary values. + /// + /// + IDictionary set_and_validate_functions(IDictionary function_dict); + + /// + /// Saves objects to a dictionary, and validates the values. + /// + /// + IDictionary set_and_validate_objects(IDictionary object_dict); + } +} diff --git a/src/TensorFlowNET.Core/Keras/Saving/TensorShapeConfig.cs b/src/TensorFlowNET.Core/Keras/Saving/TensorShapeConfig.cs new file mode 100644 index 000000000..7abcfde26 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Saving/TensorShapeConfig.cs @@ -0,0 +1,21 @@ +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Tensorflow.Keras.Saving +{ + public class TensorShapeConfig + { + [JsonProperty("class_name")] + public string ClassName { get; set; } = "TensorShape"; + [JsonProperty("items")] + public long?[] Items { get; set; } + + public static implicit operator Shape(TensorShapeConfig shape) + => shape == null ? null : new Shape(shape.Items.Select(x => x.HasValue ? x.Value : -1).ToArray()); + + public static implicit operator TensorShapeConfig(Shape shape) + => new TensorShapeConfig() { Items = shape.dims.Select(x => x == -1 ? null : x).ToArray() }; + } +} diff --git a/src/TensorFlowNET.Core/ModelSaving/SaveOptions.cs b/src/TensorFlowNET.Core/ModelSaving/SaveOptions.cs index e25537d80..45ebd884f 100644 --- a/src/TensorFlowNET.Core/ModelSaving/SaveOptions.cs +++ b/src/TensorFlowNET.Core/ModelSaving/SaveOptions.cs @@ -9,10 +9,52 @@ namespace Tensorflow.ModelSaving /// public class SaveOptions { - bool save_debug_info; + public bool save_debug_info = false; + public IList? namespace_white_list { get; set; } = null; + public IDictionary? function_aliases { get; set; } = null; + public string? experimental_io_device { get; set; } = null; + // TODO: experimental + public VariablePolicy experimental_variable_policy { get; set; } = VariablePolicy.None; + public bool experimental_custom_gradients { get; set; } = true; public SaveOptions(bool save_debug_info = false) { this.save_debug_info = save_debug_info; } } + + public class VariablePolicy + { + public string Policy { get; } + private VariablePolicy(string policy) + { + Policy = policy; + } + public static VariablePolicy None = new(null); + public static VariablePolicy SAVE_VARIABLE_DEVICES = new("save_variable_devices"); + public static VariablePolicy EXPAND_DISTRIBUTED_VARIABLES = new("expand_distributed_variables"); + + public bool save_variable_devices() + { + return this != VariablePolicy.None; + } + + /// + /// Tries to convert `obj` to a VariablePolicy instance. + /// + /// + /// + public static VariablePolicy from_obj(object obj) + { + if (obj is null) return VariablePolicy.None; + if (obj is VariablePolicy) return (VariablePolicy)obj; + var key = obj.ToString().ToLower(); + return key switch + { + null => VariablePolicy.None, + "save_variable_devices" => VariablePolicy.SAVE_VARIABLE_DEVICES, + "expand_distributed_variables" => VariablePolicy.EXPAND_DISTRIBUTED_VARIABLES, + _ => throw new ValueError($"Received invalid VariablePolicy value: {obj}.") + }; + } + } } diff --git a/src/TensorFlowNET.Core/NumPy/Axis.cs b/src/TensorFlowNET.Core/NumPy/Axis.cs index 6c7189df1..709ca9b27 100644 --- a/src/TensorFlowNET.Core/NumPy/Axis.cs +++ b/src/TensorFlowNET.Core/NumPy/Axis.cs @@ -14,20 +14,29 @@ You may obtain a copy of the License at limitations under the License. ******************************************************************************/ +using Newtonsoft.Json; using System; using System.Collections.Generic; using System.Linq; using System.Text; +using Tensorflow.Keras.Common; namespace Tensorflow { - public record Axis(params int[] axis) + [JsonConverter(typeof(CustomizedAxisJsonConverter))] + public class Axis { + public int[] axis { get; set; } public int size => axis == null ? -1 : axis.Length; public bool IsScalar { get; init; } public int this[int index] => axis[index]; + public Axis(params int[] axis) + { + this.axis = axis; + } + public static implicit operator int[]?(Axis axis) => axis?.axis; diff --git a/src/TensorFlowNET.Core/Numpy/Shape.cs b/src/TensorFlowNET.Core/Numpy/Shape.cs index bc79fefca..ecf735869 100644 --- a/src/TensorFlowNET.Core/Numpy/Shape.cs +++ b/src/TensorFlowNET.Core/Numpy/Shape.cs @@ -14,14 +14,17 @@ You may obtain a copy of the License at limitations under the License. ******************************************************************************/ +using Newtonsoft.Json; using System; using System.Collections.Generic; using System.Linq; using System.Text; +using Tensorflow.Keras.Common; using Tensorflow.NumPy; namespace Tensorflow { + [JsonConverter(typeof(CustomizedShapeJsonConverter))] public class Shape { public int ndim => _dims == null ? -1 : _dims.Length; diff --git a/src/TensorFlowNET.Core/Operations/Initializers/Constant.cs b/src/TensorFlowNET.Core/Operations/Initializers/Constant.cs index fdcb5aff0..e7e9955c0 100644 --- a/src/TensorFlowNET.Core/Operations/Initializers/Constant.cs +++ b/src/TensorFlowNET.Core/Operations/Initializers/Constant.cs @@ -14,6 +14,8 @@ You may obtain a copy of the License at limitations under the License. ******************************************************************************/ +using System.Collections.Generic; + namespace Tensorflow.Operations.Initializers { public class Constant : IInitializer @@ -22,11 +24,19 @@ public class Constant : IInitializer T value; bool _verify_shape; + private readonly Dictionary _config; + + public string ClassName => "Constant"; + public IDictionary Config => _config; + public Constant(T value, TF_DataType dtype = TF_DataType.TF_FLOAT, bool verify_shape = false) { this.value = value; this.dtype = dtype; _verify_shape = verify_shape; + + _config = new Dictionary(); + _config["value"] = this.value; } public Tensor Apply(InitializerArgs args) diff --git a/src/TensorFlowNET.Core/Operations/Initializers/GlorotUniform.cs b/src/TensorFlowNET.Core/Operations/Initializers/GlorotUniform.cs index d97d88308..def1cb7a0 100644 --- a/src/TensorFlowNET.Core/Operations/Initializers/GlorotUniform.cs +++ b/src/TensorFlowNET.Core/Operations/Initializers/GlorotUniform.cs @@ -14,10 +14,17 @@ You may obtain a copy of the License at limitations under the License. ******************************************************************************/ +using System.Collections.Generic; + namespace Tensorflow.Operations.Initializers { public class GlorotUniform : VarianceScaling { + private readonly Dictionary _config; + + public override string ClassName => "GlorotUniform"; + public override IDictionary Config => _config; + public GlorotUniform(float scale = 1.0f, string mode = "FAN_AVG", bool uniform = true, @@ -28,7 +35,8 @@ public GlorotUniform(float scale = 1.0f, seed: seed, dtype: dtype) { - + _config = new Dictionary(); + _config["seed"] = _seed; } } } diff --git a/src/TensorFlowNET.Core/Operations/Initializers/IInitializer.cs b/src/TensorFlowNET.Core/Operations/Initializers/IInitializer.cs index 50d4d5037..9748b1004 100644 --- a/src/TensorFlowNET.Core/Operations/Initializers/IInitializer.cs +++ b/src/TensorFlowNET.Core/Operations/Initializers/IInitializer.cs @@ -14,10 +14,17 @@ You may obtain a copy of the License at limitations under the License. ******************************************************************************/ +using Newtonsoft.Json; +using System.Collections.Generic; + namespace Tensorflow { public interface IInitializer { + [JsonProperty("class_name")] + string ClassName { get; } + [JsonProperty("config")] + IDictionary Config { get; } Tensor Apply(InitializerArgs args); } } diff --git a/src/TensorFlowNET.Core/Operations/Initializers/Ones.cs b/src/TensorFlowNET.Core/Operations/Initializers/Ones.cs index 02d3c93b2..3077a1e0e 100644 --- a/src/TensorFlowNET.Core/Operations/Initializers/Ones.cs +++ b/src/TensorFlowNET.Core/Operations/Initializers/Ones.cs @@ -14,12 +14,19 @@ You may obtain a copy of the License at limitations under the License. ******************************************************************************/ +using System.Collections.Generic; + namespace Tensorflow.Operations.Initializers { public class Ones : IInitializer { private TF_DataType dtype; + private readonly Dictionary _config; + + public string ClassName => "Ones"; + public IDictionary Config => new Dictionary(); + public Ones(TF_DataType dtype = TF_DataType.TF_FLOAT) { this.dtype = dtype; diff --git a/src/TensorFlowNET.Core/Operations/Initializers/Orthogonal.cs b/src/TensorFlowNET.Core/Operations/Initializers/Orthogonal.cs index 045b02c5a..492047c9f 100644 --- a/src/TensorFlowNET.Core/Operations/Initializers/Orthogonal.cs +++ b/src/TensorFlowNET.Core/Operations/Initializers/Orthogonal.cs @@ -1,4 +1,4 @@ -/***************************************************************************** +/***************************************************************************** Copyright 2023 Haiping Chen. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,6 +19,7 @@ limitations under the License. using static Tensorflow.Binding; namespace Tensorflow.Operations.Initializers; +using System.Collections.Generic; public class Orthogonal : IInitializer { @@ -31,6 +32,10 @@ public Orthogonal(float gain = 1.0f, int? seed = null) _seed = seed; } + private readonly Dictionary _config; + + public string ClassName => "Orthogonal"; + public IDictionary Config => throw new NotImplementedException(); public Tensor Apply(InitializerArgs args) { return _generate_init_val(args.Shape, args.DType == TF_DataType.DtInvalid ? TF_DataType.TF_FLOAT : args.DType); diff --git a/src/TensorFlowNET.Core/Operations/Initializers/RandomNormal.cs b/src/TensorFlowNET.Core/Operations/Initializers/RandomNormal.cs index 029b311bb..21fa7e2b2 100644 --- a/src/TensorFlowNET.Core/Operations/Initializers/RandomNormal.cs +++ b/src/TensorFlowNET.Core/Operations/Initializers/RandomNormal.cs @@ -14,6 +14,8 @@ You may obtain a copy of the License at limitations under the License. ******************************************************************************/ +using System.Collections.Generic; + namespace Tensorflow.Operations.Initializers { public class RandomNormal : IInitializer @@ -23,6 +25,11 @@ public class RandomNormal : IInitializer private int? seed; private TF_DataType dtype; + private readonly Dictionary _config; + + public string ClassName => "RandomNormal"; + public IDictionary Config => _config; + public RandomNormal(float mean = 0.0f, float stddev = 0.05f, int? seed = null, @@ -32,6 +39,11 @@ public RandomNormal(float mean = 0.0f, this.stddev = stddev; this.seed = seed; this.dtype = dtype; + + _config = new Dictionary(); + _config["mean"] = this.mean; + _config["stddev"] = this.stddev; + _config["seed"] = this.seed; } public Tensor Apply(InitializerArgs args) diff --git a/src/TensorFlowNET.Core/Operations/Initializers/RandomUniform.cs b/src/TensorFlowNET.Core/Operations/Initializers/RandomUniform.cs index a49d59212..87404708c 100644 --- a/src/TensorFlowNET.Core/Operations/Initializers/RandomUniform.cs +++ b/src/TensorFlowNET.Core/Operations/Initializers/RandomUniform.cs @@ -14,6 +14,8 @@ You may obtain a copy of the License at limitations under the License. ******************************************************************************/ +using System.Collections.Generic; + namespace Tensorflow.Operations.Initializers { public class RandomUniform : IInitializer @@ -23,12 +25,22 @@ public class RandomUniform : IInitializer private float maxval; private TF_DataType dtype; + private readonly Dictionary _config; + + public string ClassName => "RandomUniform"; + public IDictionary Config => _config; + public RandomUniform(TF_DataType dtype = TF_DataType.TF_FLOAT, float minval = -0.05f, float maxval = 0.05f, int? seed = null) { this.dtype = dtype; this.minval = minval; this.maxval = maxval; this.seed = seed; + + _config = new Dictionary(); + _config["minval"] = this.minval; + _config["maxval"] = this.maxval; + _config["seed"] = this.seed; } public Tensor Apply(InitializerArgs args) diff --git a/src/TensorFlowNET.Core/Operations/Initializers/TruncatedNormal.cs b/src/TensorFlowNET.Core/Operations/Initializers/TruncatedNormal.cs index 048c11e7a..c1c3e9996 100644 --- a/src/TensorFlowNET.Core/Operations/Initializers/TruncatedNormal.cs +++ b/src/TensorFlowNET.Core/Operations/Initializers/TruncatedNormal.cs @@ -14,6 +14,8 @@ You may obtain a copy of the License at limitations under the License. ******************************************************************************/ +using System.Collections.Generic; + namespace Tensorflow.Operations.Initializers { public class TruncatedNormal : IInitializer @@ -23,6 +25,11 @@ public class TruncatedNormal : IInitializer private int? seed; private TF_DataType dtype; + private readonly Dictionary _config; + + public string ClassName => "TruncatedNormal"; + public IDictionary Config => _config; + public TruncatedNormal(float mean = 0.0f, float stddev = 1.0f, int? seed = null, @@ -32,6 +39,10 @@ public TruncatedNormal(float mean = 0.0f, this.stddev = stddev; this.seed = seed; this.dtype = dtype; + _config = new Dictionary(); + _config["mean"] = this.mean; + _config["stddev"] = this.stddev; + _config["seed"] = this.seed; } public Tensor Apply(InitializerArgs args) diff --git a/src/TensorFlowNET.Core/Operations/Initializers/VarianceScaling.cs b/src/TensorFlowNET.Core/Operations/Initializers/VarianceScaling.cs index d313f4c9a..f104e8e83 100644 --- a/src/TensorFlowNET.Core/Operations/Initializers/VarianceScaling.cs +++ b/src/TensorFlowNET.Core/Operations/Initializers/VarianceScaling.cs @@ -15,7 +15,9 @@ limitations under the License. ******************************************************************************/ using System; +using System.Collections.Generic; using System.Linq; +using System.Linq.Expressions; namespace Tensorflow.Operations.Initializers { @@ -30,6 +32,11 @@ public class VarianceScaling : IInitializer protected int? _seed; protected TF_DataType _dtype; protected bool _uniform; + private readonly Dictionary _config; + + public virtual string ClassName => "VarianceScaling"; + + public virtual IDictionary Config => _config; public VarianceScaling(float factor = 2.0f, string mode = "FAN_IN", @@ -50,6 +57,12 @@ public VarianceScaling(float factor = 2.0f, _seed = seed; _dtype = dtype; _uniform = uniform; + + _config = new(); + _config["scale"] = _scale; + _config["mode"] = _mode; + _config["distribution"] = _distribution; + _config["seed"] = _seed; } public Tensor Apply(InitializerArgs args) diff --git a/src/TensorFlowNET.Core/Operations/Initializers/Zeros.cs b/src/TensorFlowNET.Core/Operations/Initializers/Zeros.cs index 5d045292f..c4ed25a17 100644 --- a/src/TensorFlowNET.Core/Operations/Initializers/Zeros.cs +++ b/src/TensorFlowNET.Core/Operations/Initializers/Zeros.cs @@ -14,6 +14,8 @@ You may obtain a copy of the License at limitations under the License. ******************************************************************************/ +using System.Collections.Generic; + namespace Tensorflow.Operations.Initializers { public class Zeros : IInitializer @@ -21,6 +23,9 @@ public class Zeros : IInitializer Shape shape; TF_DataType dtype; + public string ClassName => "Zeros"; + public IDictionary Config => new Dictionary(); + public Zeros(Shape shape = null, TF_DataType dtype = TF_DataType.TF_FLOAT) { this.shape = shape; diff --git a/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs b/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs index d63d0311b..2b83dd1d1 100644 --- a/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs +++ b/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs @@ -20,7 +20,9 @@ limitations under the License. using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; +using Tensorflow.Keras.Saving; using Tensorflow.Operations; +using Tensorflow.Train; using Tensorflow.Util; using static Tensorflow.Binding; @@ -75,6 +77,8 @@ public abstract class RnnCell : ILayer, RNNArgs.IRnnArgCell public Shape BatchInputShape => throw new NotImplementedException(); + public TensorShapeConfig BuildInputShape => throw new NotImplementedException(); + public TF_DataType DType => throw new NotImplementedException(); protected bool built = false; public bool Built => built; @@ -143,7 +147,7 @@ public int count_params() throw new NotImplementedException(); } - public LayerArgs get_config() + public IKerasConfig get_config() { throw new NotImplementedException(); } @@ -152,5 +156,7 @@ public void build(Shape input_shape) { throw new NotImplementedException(); } + + public Trackable GetTrackable() { throw new NotImplementedException(); } } } diff --git a/src/TensorFlowNET.Core/Operations/gen_ops.cs b/src/TensorFlowNET.Core/Operations/gen_ops.cs index 11cb6de8e..956be96b5 100644 --- a/src/TensorFlowNET.Core/Operations/gen_ops.cs +++ b/src/TensorFlowNET.Core/Operations/gen_ops.cs @@ -1,6 +1,9 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Xml.Linq; +using Tensorflow.Contexts; +using Tensorflow.Eager; using static Tensorflow.Binding; namespace Tensorflow.Operations @@ -17182,17 +17185,47 @@ public static Tensor merge_summary(Tensor[] inputs, string name = "MergeSummary" /// path in the input checkpoint_prefixes. This is useful when those paths are non /// user-facing temporary locations. /// - public static Operation merge_v2checkpoints(Tensor checkpoint_prefixes, Tensor destination_prefix, bool? delete_old_dirs = null, string name = "MergeV2Checkpoints") - { + public static Operation merge_v2_checkpoints(Tensor[] checkpoint_prefixes, Tensor destination_prefix, bool delete_old_dirs = true, bool allow_missing_files = false, string name = "MergeV2Checkpoints") + { + var ctx = tf.Context; + if (ctx.executing_eagerly()) + { + var result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo("MergeV2Checkpoints", name, + checkpoint_prefixes, destination_prefix, "delete_old_dirs", delete_old_dirs, "allow_missing_files", allow_missing_files)); + result = null; + return null; + //try + //{ + // var result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo("MergeV2Checkpoints", name, + // new object[] { checkpoint_prefixes, destination_prefix, "delete_old_dirs", delete_old_dirs, "allow_missing_files", allow_missing_files })); + // result = null; + // return null; + //} + //catch (System.Exception) + //{ + // return merge_v2_checkpoints_eager_fallback(checkpoint_prefixes, destination_prefix, delete_old_dirs: delete_old_dirs, + // allow_missing_files: allow_missing_files, name: name, ctx: ctx); + //} + } var dict = new Dictionary(); dict["checkpoint_prefixes"] = checkpoint_prefixes; dict["destination_prefix"] = destination_prefix; - if (delete_old_dirs.HasValue) - dict["delete_old_dirs"] = delete_old_dirs.Value; + dict["delete_old_dirs"] = delete_old_dirs; var op = tf.OpDefLib._apply_op_helper("MergeV2Checkpoints", name: name, keywords: dict); return op; } + //public static Operation merge_v2_checkpoints_eager_fallback(Tensor[] checkpoint_prefixes, Tensor destination_prefix, bool delete_old_dirs, bool allow_missing_files, string name, Context ctx) + //{ + // checkpoint_prefixes = ops.convert_to_tensor(checkpoint_prefixes, TF_DataType.TF_STRING); + // destination_prefix = ops.convert_to_tensor(destination_prefix, TF_DataType.TF_STRING); + // var inputs_flat = new Tensor[] { checkpoint_prefixes, destination_prefix }; + // var attrs = new object[] { "delete_old_dirs", delete_old_dirs, "allow_missing_files", allow_missing_files }; + // var result = execute.quick_execute("MergeV2Checkpoints", 0, inputs_flat, attrs, ctx, name); + // result = null; + // return null; + //} + /// /// Transforms a spectrogram into a form that's useful for speech recognition. /// @@ -24259,6 +24292,12 @@ public static (Tensor output_false, Tensor output_true) ref_switch(Tensor data, /// public static Tensor regex_full_match(Tensor input, Tensor pattern, string name = "RegexFullMatch") { + var ctx = tf.Context; + if (ctx.executing_eagerly()) + { + var result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo("RegexFullMatch", name, input, pattern)); + return result[0]; + } var dict = new Dictionary(); dict["input"] = input; dict["pattern"] = pattern; @@ -29744,6 +29783,12 @@ public static Tensor[] shape_n(Tensor[] input, TF_DataType? out_type = null, str /// public static Tensor sharded_filename(Tensor basename, Tensor shard, Tensor num_shards, string name = "ShardedFilename") { + var ctx = tf.Context; + if (ctx.executing_eagerly()) + { + var result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo("ShardedFilename", name, basename, shard, num_shards)); + return result[0]; + } var dict = new Dictionary(); dict["basename"] = basename; dict["shard"] = shard; @@ -34668,6 +34713,12 @@ public static Tensor strided_slice_grad(Tensor shape, Tensor begin, Tensor end, /// public static Tensor string_join(Tensor[] inputs, string separator = null, string name = "StringJoin") { + var ctx = tf.Context; + if (ctx.executing_eagerly()) + { + var result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo("StringJoin", name, inputs, "separator", separator)); + return result[0]; + } var dict = new Dictionary(); dict["inputs"] = inputs; if (separator != null) diff --git a/src/TensorFlowNET.Core/Operations/io_ops.cs b/src/TensorFlowNET.Core/Operations/io_ops.cs index 4f276e36c..35c5877f3 100644 --- a/src/TensorFlowNET.Core/Operations/io_ops.cs +++ b/src/TensorFlowNET.Core/Operations/io_ops.cs @@ -14,7 +14,9 @@ You may obtain a copy of the License at limitations under the License. ******************************************************************************/ +using System.Linq; using Tensorflow.Contexts; +using Tensorflow.Eager; using static Tensorflow.Binding; namespace Tensorflow @@ -23,11 +25,41 @@ public class io_ops { public Operation save_v2(Tensor prefix, string[] tensor_names, string[] shape_and_slices, Tensor[] tensors, string name = null) { + var ctx = tf.Context; + if (ctx.executing_eagerly()) + { + try + { + var result = tf.Runner.TFE_FastPathExecute( + new FastPathOpExecInfo("SaveV2", name, new object[] { prefix, tensor_names, shape_and_slices, tensors })); + result = null; + return null; + } + catch (System.Exception) + { + return save_v2_eager_fallback(prefix, tensor_names, shape_and_slices, tensors, name, ctx); + } + } var _op = tf.OpDefLib._apply_op_helper("SaveV2", name: name, args: new { prefix, tensor_names, shape_and_slices, tensors }); return _op; } + public Operation save_v2_eager_fallback(Tensor prefix, string[] tensor_names, string[] shape_and_slices, Tensor[] tensors, string name, Context ctx) + { + DataType[] attr_dtypes; + (attr_dtypes, tensors) = execute.onvert_to_mixed_eager_tensors(tensors, ctx); + prefix = ops.convert_to_tensor(prefix, TF_DataType.TF_STRING); + var tensor_names_tensor = ops.convert_to_tensor(tensor_names, TF_DataType.TF_STRING); + var shape_and_slices_tensor = ops.convert_to_tensor(shape_and_slices, TF_DataType.TF_STRING); + var inputs_flat = tensors.Concat(new Tensor[] { prefix, tensor_names_tensor, shape_and_slices_tensor }).ToArray(); + var attrs = new object[] { "dtypes", attr_dtypes }; + + var result = execute.quick_execute("SaveV2", 0, inputs_flat, attrs, ctx, name); + result = null; + return null; + } + public Tensor[] restore_v2(Tensor prefix, string[] tensor_names, string[] shape_and_slices, TF_DataType[] dtypes, string name = null) { var _op = tf.OpDefLib._apply_op_helper("RestoreV2", name: name, args: new { prefix, tensor_names, shape_and_slices, dtypes }); diff --git a/src/TensorFlowNET.Core/Operations/resource_variable_ops.cs b/src/TensorFlowNET.Core/Operations/resource_variable_ops.cs index ee751acf4..1b1fa0037 100644 --- a/src/TensorFlowNET.Core/Operations/resource_variable_ops.cs +++ b/src/TensorFlowNET.Core/Operations/resource_variable_ops.cs @@ -17,6 +17,9 @@ limitations under the License. using System; using System.Linq; using Tensorflow.Framework; +using Tensorflow.ModelSaving; +using Tensorflow.Train; +using Tensorflow.Variables; using static Tensorflow.CppShapeInferenceResult.Types; namespace Tensorflow @@ -38,6 +41,11 @@ public static bool is_resource_variable(IVariableV1 var) { return var is ResourceVariable; } + + public static bool is_resource_variable(Trackable var) + { + return var is BaseResourceVariable; + } /// /// Creates a variable handle with information to do shape inference. @@ -171,5 +179,57 @@ private static HandleData get_eager_safe_handle_data(Tensor handle) return HandleData.Parser.ParseFrom(handle.BufferToArray()); } } + + /// + /// Copies an existing variable to a new graph, with no initializer. + /// + /// + public static UninitializedVariable copy_to_graph_uninitialized(ResourceVariable variable) + { + var new_variable = new UninitializedVariable( + trainable: variable.Trainable, + shape: variable.shape, + dtype: variable.dtype, + name: variable.SharedName, + aggregation: variable.Aggregation, + extra_handle_data: null); + new_variable._maybe_initialize_trackable(); + return new_variable; + } + + /// + /// Writes additional information of the variable into the SavedObject proto. + /// + /// + /// + /// + /// + public static void write_object_proto_for_resource_variable(BaseResourceVariable resource_variable, SavedObject proto, SaveOptions options, bool enforcing_naming = true) + { + // lack of API: `proto.Variable.SetInParent()`. + if(enforcing_naming && !resource_variable.Name.EndsWith(":0")) + { + throw new ValueError($"Cowardly refusing to save variable {resource_variable.Name} because of " + + $"unexpected suffix in the name (expected ':0') which won't be restored."); + } + if(proto.Variable is null) + { + proto.Variable = new SavedVariable(); + } + proto.Variable.Name = meta_graph.op_name(resource_variable.Name); + proto.Variable.Trainable = resource_variable.Trainable; + proto.Variable.Dtype = resource_variable.dtype.as_datatype_enum(); + // TODO: lack of API `proto.Variable.Synchronization = resource_variable.synchronization.value`. + proto.Variable.Aggregation = resource_variable.Aggregation; + proto.Variable.Shape = resource_variable.shape.as_proto(); + + if (options.experimental_variable_policy.save_variable_devices()) + { + if (!string.IsNullOrEmpty(resource_variable.Device)) + { + proto.Variable.Device = resource_variable.Device; + } + } + } } } diff --git a/src/TensorFlowNET.Core/Protobuf/SavedObjectGraph.cs b/src/TensorFlowNET.Core/Protobuf/SavedObjectGraph.cs index 9d3e854ac..f2597574b 100644 --- a/src/TensorFlowNET.Core/Protobuf/SavedObjectGraph.cs +++ b/src/TensorFlowNET.Core/Protobuf/SavedObjectGraph.cs @@ -156,7 +156,7 @@ public SavedObjectGraph Clone() { /// Nodes[0] is considered the root node. /// [global::System.Diagnostics.DebuggerNonUserCodeAttribute] - public pbc::RepeatedField Nodes { + public pbc::RepeatedField Nodes { get { return nodes_; } } @@ -286,6 +286,7 @@ public SavedObject() { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public SavedObject(SavedObject other) : this() { children_ = other.children_.Clone(); + dependencies_ = other.dependencies_.Clone(); slotVariables_ = other.slotVariables_.Clone(); saveableObjects_ = other.saveableObjects_.Clone(); switch (other.KindCase) { @@ -328,6 +329,7 @@ public SavedObject Clone() { private static readonly pb::FieldCodec _repeated_children_codec = pb::FieldCodec.ForMessage(10, global::Tensorflow.TrackableObjectGraph.Types.TrackableObject.Types.ObjectReference.Parser); private readonly pbc::RepeatedField children_ = new pbc::RepeatedField(); + private readonly pbc::RepeatedField dependencies_ = new pbc::RepeatedField(); /// /// Objects which this object depends on: named edges in the dependency /// graph. @@ -338,6 +340,11 @@ public SavedObject Clone() { public pbc::RepeatedField Children { get { return children_; } } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public pbc::RepeatedField Dependencies { + get { return dependencies_; } + } /// Field number for the "slot_variables" field. public const int SlotVariablesFieldNumber = 3; @@ -617,6 +624,7 @@ public void MergeFrom(SavedObject other) { return; } children_.Add(other.children_); + dependencies_.Add(other.dependencies_); slotVariables_.Add(other.slotVariables_); saveableObjects_.Add(other.saveableObjects_); switch (other.KindCase) { diff --git a/src/TensorFlowNET.Core/Protobuf/TrackableObjectGraph.cs b/src/TensorFlowNET.Core/Protobuf/TrackableObjectGraph.cs index 3aa747c20..fb197eca2 100644 --- a/src/TensorFlowNET.Core/Protobuf/TrackableObjectGraph.cs +++ b/src/TensorFlowNET.Core/Protobuf/TrackableObjectGraph.cs @@ -198,6 +198,22 @@ public sealed partial class TrackableObject : pb::IMessage { public TrackableObject() { OnConstruction(); } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public TrackableObject(pbc::RepeatedField slot) { + OnConstruction(); + slotVariables_ = slot; + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public TrackableObject(pbc::RepeatedField slot, + pbc::RepeatedField children + ) + { + OnConstruction(); + slotVariables_ = slot; + children_ = children; + } partial void OnConstruction(); diff --git a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj index a7db6eee1..ede72a6ae 100644 --- a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj +++ b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj @@ -108,6 +108,7 @@ https://tensorflownet.readthedocs.io + diff --git a/src/TensorFlowNET.Core/Tensors/dtypes.cs b/src/TensorFlowNET.Core/Tensors/dtypes.cs index 372ac6762..deeb9e4b5 100644 --- a/src/TensorFlowNET.Core/Tensors/dtypes.cs +++ b/src/TensorFlowNET.Core/Tensors/dtypes.cs @@ -202,6 +202,24 @@ public static string as_numpy_name(this TF_DataType type) _ => type.ToString() }; + public static string as_python_name(this TF_DataType type) + => type switch + { + TF_DataType.TF_STRING => "str", + TF_DataType.TF_UINT8 => "uint8", + TF_DataType.TF_INT8 => "int8", + TF_DataType.TF_UINT32 => "uint32", + TF_DataType.TF_INT32 => "int32", + TF_DataType.TF_UINT64 => "uint64", + TF_DataType.TF_INT64 => "int64", + TF_DataType.TF_FLOAT => "float32", + TF_DataType.TF_DOUBLE => "float64", + TF_DataType.TF_BOOL => "bool", + TF_DataType.TF_RESOURCE => "resource", + TF_DataType.TF_VARIANT => "variant", + _ => type.ToString() + }; + public static int get_datatype_size(this TF_DataType type) => type.as_base_dtype() switch { diff --git a/src/TensorFlowNET.Core/Training/AutoTrackable.cs b/src/TensorFlowNET.Core/Training/AutoTrackable.cs index d2198e37e..4ba3e4074 100644 --- a/src/TensorFlowNET.Core/Training/AutoTrackable.cs +++ b/src/TensorFlowNET.Core/Training/AutoTrackable.cs @@ -1,6 +1,71 @@ -namespace Tensorflow.Train +using System.Collections.Generic; +using System.Linq; +using Tensorflow.Functions; +using Tensorflow.Keras.Saving.SavedModel; +using Tensorflow.Operations.Activation; +using static Tensorflow.Binding; + +namespace Tensorflow.Train { - public abstract class AutoTrackable : Trackable + public class AutoTrackable : Trackable { + public void _delete_tracking(string name) + { + _maybe_initialize_trackable(); + if (_unconditional_dependency_names.ContainsKey(name)) + { + _unconditional_dependency_names.Remove(name); + for (int i = _unconditional_checkpoint_dependencies.Count - 1; i >= 0; i--) + { + if (_unconditional_checkpoint_dependencies[i].Name == name) + { + _unconditional_checkpoint_dependencies.RemoveAt(i); + } + } + } + } + + public override IDictionary _trackable_children(SaveType save_type, IDictionary>? cache = null) + { + if(save_type != SaveType.SAVEDMODEL) + { + return base._trackable_children(save_type, cache); + } + + Dictionary functions = new(); + // TODO: process of logs. + var properties = this.GetType().GetProperties(); + foreach ( var property in properties ) + { + if(property.PropertyType == typeof(Function) || property.PropertyType == typeof(ConcreteFunction)) + { + string name = property.Name; + object value = property.GetValue(this, null); + functions[name] = (Trackable)value; + } + } + + // TODO: process the type `core_types.GenericFunction`. + + Dictionary children = new(); + foreach(var pair in CheckpointDependencies) + { + var name = pair.Name; + var child = pair.Refer; + if(child is ConcreteFunction) // or Generic function + { + continue; + } + if(functions.ContainsKey(name) && functions[name] != child) + { + throw new ValueError($"Can't save object because it has multiple children with the same " + + $"name. Object: {this}, attribute name: {name}, child 1: " + + $"{child}, child 2: {functions[name]}"); + } + children[name] = child; + } + + return children.Concat(functions).ToDictionary(x => x.Key, x => x.Value); + } } } diff --git a/src/TensorFlowNET.Core/Training/IWithTrackable.cs b/src/TensorFlowNET.Core/Training/IWithTrackable.cs new file mode 100644 index 000000000..87eda8795 --- /dev/null +++ b/src/TensorFlowNET.Core/Training/IWithTrackable.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Train; + +namespace Tensorflow.Training +{ + public interface IWithTrackable + { + Trackable GetTrackable(); + } +} diff --git a/src/TensorFlowNET.Core/Training/LayerUtils.cs b/src/TensorFlowNET.Core/Training/LayerUtils.cs new file mode 100644 index 000000000..211419651 --- /dev/null +++ b/src/TensorFlowNET.Core/Training/LayerUtils.cs @@ -0,0 +1,9 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Train; + +namespace Tensorflow.Training +{ + +} diff --git a/src/TensorFlowNET.Core/Training/Optimizer.cs b/src/TensorFlowNET.Core/Training/Optimizer.cs index f985c6566..e656fe96d 100644 --- a/src/TensorFlowNET.Core/Training/Optimizer.cs +++ b/src/TensorFlowNET.Core/Training/Optimizer.cs @@ -351,7 +351,7 @@ public virtual void _prepare() /// /// /// - protected IVariableV1 get_slot(IVariableV1 var, string name) + internal IVariableV1 get_slot(IVariableV1 var, string name) { var named_slots = _slots.ContainsKey(name) ? _slots[name] : null; if (named_slots == null) @@ -360,6 +360,11 @@ protected IVariableV1 get_slot(IVariableV1 var, string name) return named_slots.ContainsKey(_var_key(var)) ? named_slots[_var_key(var)] : null; } + internal IEnumerable get_slot_names() + { + return _slots.Keys; + } + private string _var_key(IVariableV1 var) { return $"{var.Op.graph.graph_key}.{var.Op.name}"; diff --git a/src/TensorFlowNET.Core/Training/Saving/ResourceVariableSaveable.cs b/src/TensorFlowNET.Core/Training/Saving/ResourceVariableSaveable.cs index 167c635a8..2d23a325f 100644 --- a/src/TensorFlowNET.Core/Training/Saving/ResourceVariableSaveable.cs +++ b/src/TensorFlowNET.Core/Training/Saving/ResourceVariableSaveable.cs @@ -14,6 +14,8 @@ You may obtain a copy of the License at limitations under the License. ******************************************************************************/ +using static Tensorflow.Binding; + namespace Tensorflow { public class ResourceVariableSaveable : MySaveableObject @@ -35,6 +37,32 @@ public ResourceVariableSaveable(Tensor var, string slice_spec, string name) this.name = name; } + public ResourceVariableSaveable(BaseResourceVariable var, string slice_spec, string name) + { + _var_device = var.Device; + _var_shape = var.shape; + + Tensor _read_variable_closure(BaseResourceVariable v) + { + tf.device(v.Device); + if(tf.Context.executing_eagerly() && !((bool)v.is_initialized().numpy())) + { + return null; + } + var x = v.read_value_no_copy(); + tf.device("/device:CPU:0"); + return array_ops.identity(x); + } + + this.handle_op = var.Handle; + var tensor = _read_variable_closure(var); + + var spec = new SaveSpec(tensor, slice_spec, name, dtype: var.dtype); + _op = var; + specs = new SaveSpec[] { spec }; + this.name = name; + } + public override Operation restore(Tensor[] restored_tensors, Shape[] restored_shapes = null) { var restored_tensor = restored_tensors[0]; diff --git a/src/TensorFlowNET.Core/Training/Saving/SaveSpec.cs b/src/TensorFlowNET.Core/Training/Saving/SaveSpec.cs index 1ae912ce6..393a6a981 100644 --- a/src/TensorFlowNET.Core/Training/Saving/SaveSpec.cs +++ b/src/TensorFlowNET.Core/Training/Saving/SaveSpec.cs @@ -28,7 +28,7 @@ public class SaveSpec public string slice_spec => _slice_spec; private string _name; - public string name => _name; + public string name { get => _name; set => _name = value; } private TF_DataType _dtype; public TF_DataType dtype => _dtype; diff --git a/src/TensorFlowNET.Core/Training/Saving/SaveableObject.cs b/src/TensorFlowNET.Core/Training/Saving/SaveableObject.cs index c86075f86..1309a6174 100644 --- a/src/TensorFlowNET.Core/Training/Saving/SaveableObject.cs +++ b/src/TensorFlowNET.Core/Training/Saving/SaveableObject.cs @@ -14,11 +14,31 @@ You may obtain a copy of the License at limitations under the License. ******************************************************************************/ +using Tensorflow.Checkpoint; + namespace Tensorflow { public class MySaveableObject { - public Tensor op; + protected Maybe _op; + public Tensor op + { + get + { + if(_op.TryGet(out var tensor)) + { + return tensor; + } + else + { + throw new TypeError("The _op is not a tensor."); + } + } + set + { + _op = value; + } + } public SaveSpec[] specs; public string name; public string device; @@ -35,7 +55,7 @@ public MySaveableObject(Tensor var, string slice_spec, string name) public MySaveableObject(Tensor op, SaveSpec[] specs, string name) { - this.op = op; + this._op = op; this.specs = specs; this.name = name; } @@ -48,4 +68,18 @@ public virtual Operation restore(Tensor[] restored_tensors, Shape[] restored_sha validate_shape: restored_shapes == null && op.shape.IsFullyDefined); } } + + public class NoRestoreSaveable: MySaveableObject + { + public NoRestoreSaveable(Tensor tensor, string name, TF_DataType dtype = TF_DataType.DtInvalid, string? device = null) : base(tensor, + new SaveSpec[] { new SaveSpec(tensor, "", name, dtype) }, name) + { + + } + + public override Operation restore(Tensor[] restored_tensors, Shape[] restored_shapes = null) + { + return control_flow_ops.no_op(); + } + } } diff --git a/src/TensorFlowNET.Core/Training/Saving/SavedModel/AssetInfo.cs b/src/TensorFlowNET.Core/Training/Saving/SavedModel/AssetInfo.cs new file mode 100644 index 000000000..d10257822 --- /dev/null +++ b/src/TensorFlowNET.Core/Training/Saving/SavedModel/AssetInfo.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; + +namespace Tensorflow; + +public record class AssetInfo +( + List asset_defs, + Dictionary asset_initializers_by_resource, + Dictionary asset_filename_map, + Dictionary asset_index +); diff --git a/src/TensorFlowNET.Core/Training/Saving/SavedModel/AugmentedGraphView.cs b/src/TensorFlowNET.Core/Training/Saving/SavedModel/AugmentedGraphView.cs new file mode 100644 index 000000000..a91933357 --- /dev/null +++ b/src/TensorFlowNET.Core/Training/Saving/SavedModel/AugmentedGraphView.cs @@ -0,0 +1,133 @@ +using System; +using Tensorflow.Checkpoint; +using Tensorflow.Train; +using System.Collections.Generic; +using System.Linq; +using Tensorflow.Functions; +using Tensorflow.Keras.Saving.SavedModel; + +namespace Tensorflow; + +public class AugmentedGraphView: ObjectGraphView +{ + private Dictionary> _children_cache; + private Dictionary> _serialization_cache; + private List _untraces_functions; + private Dictionary _wrapped_functions; + public AugmentedGraphView(Trackable root): base(root) + { + _children_cache= new Dictionary>(); + _serialization_cache = new Dictionary>(); + _untraces_functions = new List(); + _wrapped_functions = new Dictionary(); + } + + public void set_signature(SignatureMap signature_map, IDictionary wrapped_functions) + { + list_children(Root); + var name = SignatureSerializationUtils.SIGNATURE_ATTRIBUTE_NAME; + if (!_children_cache.ContainsKey(Root)) + { + _children_cache[Root] = new Dictionary(); + } + _children_cache[Root][name] = signature_map; + _wrapped_functions = _wrapped_functions.Concat(wrapped_functions).ToDictionary(x => x.Key, x => x.Value); + } + + public override List list_children(Trackable obj, SaveType save_type = SaveType.SAVEDMODEL, IDictionary>? serialization_cache = null) + { + if(serialization_cache is not null) + { + throw new ValueError("Serialization cache should not be passed to `AugmentedGraphView.list_children`, please either remove the parameter or use `ObjectGraphView.list_children`."); + } + + if (!_children_cache.ContainsKey(obj)) + { + Dictionary children = new Dictionary(); + _children_cache[obj] = children; + foreach (var pair in base.list_children(obj, SaveType.SAVEDMODEL, _serialization_cache)) + { + var name = pair.Name; + var child = pair.Refer; + if(child is ConcreteFunction) + { + child = maybe_uncache_variable_captures((ConcreteFunction)child); + } + children[name] = child; + } + + if (obj is Function && children.Count == 0) + { + _untraces_functions.Add(((Function)obj).Name); + } + } + + List res = new(); + foreach(var pair in _children_cache[obj]) + { + res.Add(new TrackableReference(pair.Key, pair.Value)); + } + + return res; + } + + private ConcreteFunction maybe_uncache_variable_captures(ConcreteFunction concrete_function) + { + if (_wrapped_functions.ContainsKey(concrete_function)) + { + return _wrapped_functions[concrete_function]; + } + // skip the process here because of lack of feature. + // In the future, we may add an attribute which could specify if the variable is supposed to be cached. + //foreach(var capture in concrete_function.CapturedInputs) + //{ + + //} + return concrete_function; + } + + public override (IList, IDictionary>) breadth_first_traversal() + { + Trackable get_merged_trackable(Trackable x) + { + // TODO: complete it with new definitions `Asset` and `TrackableConstant`. + return x; + } + var trackable_objects = base.breadth_first_traversal(); + + foreach(var obj in _children_cache.Keys) + { + // skip the deletion of cache (maybe do it later). + foreach(var pair in _children_cache[obj]) + { + _children_cache[obj][pair.Key] = get_merged_trackable(pair.Value); + } + } + + return base.breadth_first_traversal(); + } + + public List<(string, Trackable)> list_dependencies(Trackable obj) + { + IDictionary children; + if (!_children_cache.ContainsKey(obj)) + { + children= new Dictionary(); + } + else + { + children= _children_cache[obj]; + } + List<(string, Trackable)> res = new(); + foreach(var pair in obj.deserialization_dependencies(children)) + { + res.Add((pair.Key, pair.Value)); + } + return res; + } + + public Trackable get_child(Trackable obj, string name) + { + return _children_cache[obj][name]; + } +} diff --git a/src/TensorFlowNET.Core/Training/Saving/SavedModel/Constants.cs b/src/TensorFlowNET.Core/Training/Saving/SavedModel/Constants.cs new file mode 100644 index 000000000..726f6cfd4 --- /dev/null +++ b/src/TensorFlowNET.Core/Training/Saving/SavedModel/Constants.cs @@ -0,0 +1,33 @@ +namespace Tensorflow; + +public static class Constants +{ + public static readonly string ASSETS_DIRECTORY = "assets"; + public static readonly string ASSETS_KEY = "saved_model_assets"; + + public static readonly string DEBUG_DIRECTORY = "debug"; + + public static readonly string DEBUG_INFO_FILENAME_PB = "saved_model_debug_info.pb"; + + public static readonly string EXTRA_ASSETS_DIRECTORY = "assets.extra"; + + public static readonly string FINGERPRINT_FILENAME = "fingerprint.pb"; + + public static readonly string INIT_OP_SIGNATURE_KEY = "__saved_model_init_op"; + + public static readonly string LEGACY_INIT_OP_KEY = "legacy_init_op"; + + public static readonly string MAIN_OP_KEY = "saved_model_main_op"; + + public static readonly string SAVED_MODEL_FILENAME_PB = "saved_model.pb"; + public static readonly string SAVED_MODEL_FILENAME_PBTXT = "saved_model.pbtxt"; + + public static readonly int SAVED_MODEL_SCHEMA_VERSION = 1; + + public static readonly string TRAIN_OP_KEY = "saved_model_train_op"; + + public static readonly string TRAIN_OP_SIGNATURE_KEY = "__saved_model_train_op"; + + public static readonly string VARIABLES_DIRECTORY = "variables"; + public static readonly string VARIABLES_FILENAME = "variables"; +} diff --git a/src/TensorFlowNET.Core/Training/Saving/SavedModel/RevivedTypes.cs b/src/TensorFlowNET.Core/Training/Saving/SavedModel/RevivedTypes.cs new file mode 100644 index 000000000..fe0403c30 --- /dev/null +++ b/src/TensorFlowNET.Core/Training/Saving/SavedModel/RevivedTypes.cs @@ -0,0 +1,17 @@ +using Tensorflow.Train; + +namespace Tensorflow; + +public class RevivedTypes +{ + /// + /// Create a SavedUserObject from a trackable object. + /// + /// + /// + public static SavedUserObject? serialize(Trackable obj) + { + // TODO: complete the implementation. + return null; + } +} diff --git a/src/TensorFlowNET.Core/Training/Saving/SavedModel/SaveType.cs b/src/TensorFlowNET.Core/Training/Saving/SavedModel/SaveType.cs new file mode 100644 index 000000000..8dd4f008f --- /dev/null +++ b/src/TensorFlowNET.Core/Training/Saving/SavedModel/SaveType.cs @@ -0,0 +1,9 @@ +using System; + +namespace Tensorflow; + +public enum SaveType +{ + SAVEDMODEL, + CHECKPOINT +} diff --git a/src/TensorFlowNET.Core/Training/Saving/SavedModel/SaveableView.cs b/src/TensorFlowNET.Core/Training/Saving/SavedModel/SaveableView.cs new file mode 100644 index 000000000..1be54287e --- /dev/null +++ b/src/TensorFlowNET.Core/Training/Saving/SavedModel/SaveableView.cs @@ -0,0 +1,299 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Tensorflow.Checkpoint; +using Tensorflow.Contexts; +using Tensorflow.Functions; +using Tensorflow.ModelSaving; +using Tensorflow.Train; +using Tensorflow.Training; +using pbc = global::Google.Protobuf.Collections; +using static Tensorflow.Binding; +using Tensorflow.Training.Saving.SavedModel; + +namespace Tensorflow; + +public class SaveableView +{ + private AugmentedGraphView _augmented_graph_view; + private SaveOptions _options; + private IList _trackable_objects; + private List _nodes; + private IDictionary> _node_paths; + private IDictionary _node_ids; + private IDictionary> + _slot_variables; + private IDictionary _object_names; + private List _gradient_functions; // to be completed + private List _gradient_defs; // to be completed + private List _concrete_functions; + private Dictionary _captured_tensor_node_ids; + private Dictionary> _saveable_objects_map; + private Dictionary _obj_to_registered_saver; + + public AugmentedGraphView AugmentedGraphView + { + get => _augmented_graph_view; + } + + public Trackable Root + { + get => _nodes[0]; + } + public List Nodes + { + get => _nodes; + } + public IDictionary NodeIds + { + get => _node_ids; + } + public List GradientDefs + { + get => _gradient_defs; + } + public IDictionary> NodePaths + { + get => _node_paths; + } + public SaveableView(AugmentedGraphView augmented_graph_view, SaveOptions options) + { + _augmented_graph_view = augmented_graph_view; + _options = options; + + (_trackable_objects, _node_paths, _node_ids, _slot_variables, _object_names) = + CheckPointUtils.objects_ids_and_slot_variables_and_paths(_augmented_graph_view); + + // TODO: deal with untraced functions. + + initialize_save_and_restore_functions(); + initialize_nodes_and_concrete_functions(); + + _captured_tensor_node_ids = new(); + } + + private void initialize_save_and_restore_functions() + { + // TODO: deal with the return value of `get_checkpoint_factories_and_keys`. + var (checkpoint_factory_map, registered_savers) = SaveUtilV1.get_checkpoint_factories_and_keys(_object_names); + // skip the process of registered savers and the generation of saveable_objects_map and _obj_to_registered_saver. + _obj_to_registered_saver = new(); + _saveable_objects_map = new(); + } + + private void initialize_nodes_and_concrete_functions() + { + _nodes = _trackable_objects.ToList().ConvertAll(x => x); // deep copy + _gradient_functions = new(); + _gradient_defs = new(); + + // TODO: deal with the condition that obj in `_saveable_objects_map`. + // foreach (var obj in _nodes) + // { + // + // } + + foreach (var obj in _nodes) + { + if (obj is ConcreteFunction) + { + _concrete_functions.Add((ConcreteFunction)obj); + } + } + } + + public List get_concrete_resource_initializers() + { + // TODO: complete the implementation. + return new List(); + } + + public (Dictionary, Dictionary, AssetInfo) map_resources() + { + Debug.Assert(!tf.Context.executing_eagerly()); + + Dictionary object_map = new(); + Dictionary tensor_map = new(); + + AssetInfo assetInfo = new(new List(), new Dictionary(), + new Dictionary(), new Dictionary()); + + foreach (var node_id in dependency_sorted_node_ids()) + { + var obj = _nodes[node_id]; + var tensors = obj.export_to_saved_model_graph(object_map, tensor_map, _options); + // TODO: deal with Asset (if obj is Asset) + foreach (var tensor in tensors) + { + _captured_tensor_node_ids[tensor] = node_id; + } + } + + return (object_map, tensor_map, assetInfo); + } + + /// + /// Returns topologically sorted nodes, sorted by dependencies. + /// + public List dependency_sorted_node_ids() + { + Dictionary> dependency_map = new(); + foreach (var node in _nodes) + { + var node_id = _node_ids[node]; + List deps = new List(); + dependency_map.Add(node_id, deps); + + // TODO: deal with captured tensor. + + foreach (var (_, dep) in _augmented_graph_view.list_dependencies(node)) + { + if (!_node_ids.ContainsKey(dep)) + { + var node_path = TrackableUtils.pretty_print_node_path(_node_paths[node]); + throw new ValueError( + $"Found an untracked dependency. Object {node_path} depends on {dep}, " + + $"but this dependency isn't listed as a child. Please track this child by " + + $"overriding `_trackable_children` or use `._track_trackable`."); + } + deps.Add(_node_ids[dep]); + } + } + + try + { + return TrackableUtils.order_by_dependency(dependency_map); + } + catch (TrackableUtils.CyclicDependencyError err) + { + List pretty_printed_nodes = new(); + List pretty_printed_dependencies = new(); + + foreach (var pair in err.LeftOverDependencyMap) + { + var x = pair.Key; + var deps = pair.Value; + var node_path = TrackableUtils.pretty_print_node_path(_node_paths[_nodes[x]]); + pretty_printed_nodes.Add($"\tNode {x.ToString()} = {node_path} (type {_nodes[x]})"); + pretty_printed_dependencies.Add( + $"\tNode {x.ToString()} depends on nodes [{string.Join(", ", deps.Select(x => x.ToString()))}]"); + } + + throw new ValueError($"There is one or more dependency cycle in the saved Trackable object. " + + $"Saving cannot continue until this cycle is resolved." + + $"\n>> Unresolved nodes:\n{string.Join("\n", pretty_printed_nodes)}" + + $"\n>> Unresolved cyclic dependencies:\n{string.Join("\n", pretty_printed_dependencies)}"); + } + } + + /// + /// Corresponding to tensorflow/python/saved_model/save.py/_serialize_object_graph + /// + /// + /// + public SavedObjectGraph serialize_object_graph(IDictionary asset_file_def_index) + { + SavedObjectGraph proto = new(); + fill_object_graph_proto(proto); + + // TODO: complete the process of concrete functions. + + int cnt = Math.Min(_nodes.Count, proto.Nodes.Count); + for (int i = 0; i < cnt; i++) + { + var obj = _nodes[i]; + var obj_proto = proto.Nodes[i]; + write_object_proto(obj, obj_proto, asset_file_def_index, x => _augmented_graph_view.list_children(x)); + } + + return proto; + } + + private static void write_object_proto(Trackable obj, SavedObject proto, + IDictionary asset_file_def_index, Func> list_children_fn) + { + // skip the process of type Asset + if (resource_variable_ops.is_resource_variable(obj)) + { + var options = SaveContext.get_save_options(); + (obj as BaseResourceVariable).write_object_proto(proto, options); + } + else if (obj is Function) + { + // TODO: complete it. + throw new NotImplementedException(); + } + else if (obj is ConcreteFunction) + { + // TODO: complete it. + throw new NotImplementedException(); + } + // skip the process of type `_CapturedTensor` and `CapturableResource`. + else + { + var registered_type_proto = RevivedTypes.serialize(obj); + if (registered_type_proto is null) + { + registered_type_proto = new SavedUserObject() + { + Identifier = obj.ObjectIdentifier, + Version = new VersionDef() + { + Producer = 1, + MinConsumer = 1, + BadConsumers = { } + } + }; + } + + proto.UserObject = new SavedUserObject(registered_type_proto); + } + + // TODO: try get the registered_name from `registration`. + } + + public void fill_object_graph_proto(SavedObjectGraph proto) + { + for (int node_id = 0; node_id < _nodes.Count; node_id++) + { + var node = _nodes[node_id]; + Debug.Assert(_node_ids[node] == node_id); + SavedObject object_proto = new(); + if (_slot_variables.TryGetValue(node, out var value)) + { + object_proto.SlotVariables.AddRange(value); + } + // skip the check of type `_CapturedTensor` + foreach (var child in _augmented_graph_view.list_children(node)) + { + var child_proto = new TrackableObjectGraph.Types.TrackableObject.Types.ObjectReference(); + child_proto.NodeId = _node_ids[child.Refer]; + child_proto.LocalName = child.Name; + object_proto.Children.Add(child_proto); + } + + foreach (var pair in _augmented_graph_view.list_dependencies(node)) + { + var child_proto = new TrackableObjectGraph.Types.TrackableObject.Types.ObjectReference(); + child_proto.NodeId = _node_ids[pair.Item2]; + child_proto.LocalName = pair.Item1; + object_proto.Dependencies.Add(child_proto); + } + + if (_saveable_objects_map.ContainsKey(node)) + { + // TODO: complete it. + throw new NotImplementedException(); + } + else if(_obj_to_registered_saver.ContainsKey(node)) + { + // TODO: complete it. + // We now skip it for the lack of `SavedObject.registered_saver` API. + throw new NotImplementedException(); + } + + proto.Nodes.Add(object_proto); + } + } +} diff --git a/src/TensorFlowNET.Core/Training/Saving/SavedModel/TagConstants.cs b/src/TensorFlowNET.Core/Training/Saving/SavedModel/TagConstants.cs new file mode 100644 index 000000000..6aa1fbde1 --- /dev/null +++ b/src/TensorFlowNET.Core/Training/Saving/SavedModel/TagConstants.cs @@ -0,0 +1,10 @@ +namespace Tensorflow; + +public static class TagConstants +{ + public static readonly string SERVING = "serve"; + public static readonly string TRAINING = "train"; + public static readonly string EVAL = "eval"; + public static readonly string GPU = "gpu"; + public static readonly string TPU = "tpu"; +} diff --git a/src/TensorFlowNET.Core/Training/Saving/SavedModel/builder.cs b/src/TensorFlowNET.Core/Training/Saving/SavedModel/builder.cs new file mode 100644 index 000000000..dbbab91d8 --- /dev/null +++ b/src/TensorFlowNET.Core/Training/Saving/SavedModel/builder.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using static Tensorflow.Binding; + +namespace Tensorflow; + +public class BuilderUtils +{ + public static void copy_assets_to_destination_dir(IDictionary asset_filename_map, + string destination_dir, HashSet? saved_files = null) + { + if (saved_files is null) saved_files = new HashSet(); + + var asset_destination_dir = SavedModelUtils.get_or_create_assets_dir(destination_dir); + + // TODO: complete the implementation of this function. + if (asset_filename_map is not null && asset_filename_map.Count > 0) + { + throw new NotImplementedException(); + } + } +} diff --git a/src/TensorFlowNET.Core/Training/Saving/SavedModel/save.cs b/src/TensorFlowNET.Core/Training/Saving/SavedModel/save.cs new file mode 100644 index 000000000..94760e3df --- /dev/null +++ b/src/TensorFlowNET.Core/Training/Saving/SavedModel/save.cs @@ -0,0 +1,269 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using Google.Protobuf; +using Tensorflow.Checkpoint; +using Tensorflow.Functions; +using Tensorflow.ModelSaving; +using Tensorflow.Train; +using Tensorflow.Exceptions; +using static Tensorflow.Binding; +using Tensorflow.Training.Saving.SavedModel; + +namespace Tensorflow; + +public static partial class SavedModelUtils +{ + private static readonly IEnumerable byte_swappable = new List() + { + dtypes.float16, dtypes.float32, dtypes.float64, TF_DataType.TF_BFLOAT16, + dtypes.complex64, dtypes.complex128, TF_DataType.TF_UINT16, dtypes.uint32, + dtypes.uint64, TF_DataType.TF_INT16, dtypes.int32, dtypes.int64, TF_DataType.TF_QINT16, + TF_DataType.TF_QUINT16, TF_DataType.TF_QINT32 + }.Select(x => (int)x); + + public static (IList, IDictionary>) save_and_return_nodes(Trackable obj, + string export_dir, ConcreteFunction? signatures, SaveOptions? options = null, bool experimental_skip_checkpoint = false) + { + if (options is null) + { + options = new SaveOptions(); + } + + var saved_model = new Tensorflow.SavedModel(); + var meta_graph_def = new MetaGraphDef(); + saved_model.MetaGraphs.Add(meta_graph_def); + + var (_, exported_graph, object_saver, asset_info, saved_nodes, node_paths) = + _build_meta_graph(obj, signatures, options, meta_graph_def); + saved_model.SavedModelSchemaVersion = Tensorflow.Constants.SAVED_MODEL_SCHEMA_VERSION; + + if (!experimental_skip_checkpoint) + { + SavedModelUtils.get_or_create_variables_dir(export_dir); + CheckpointOptions ckpt_options = new(options.experimental_io_device); + object_saver.save(SavedModelUtils.get_variables_path(export_dir), options:ckpt_options); + } + BuilderUtils.copy_assets_to_destination_dir(asset_info.asset_filename_map, export_dir); + + if (tf.Context.executing_eagerly()) + { + // tensorflow python has a check of `context.async_wait()` here. + } + + // TODO: deal with `pywrap_saved_model.Save(export_dir)`. + + var saved_model_serialized = saved_model.ToString(); + + // This is a state depending on some py-c APIs. Here we temporarily set it as `true`. + if (true) + { + var fingerprint_path = Path.Combine(tf.compat.as_str(export_dir), + tf.compat.as_str(Constants.FINGERPRINT_FILENAME)); + // TODO: add c api and complete the fingerprint def. + var fingerprint_proto = ""; + File.WriteAllText(fingerprint_path, fingerprint_proto); + } + + var path = Path.Combine(tf.compat.as_str(export_dir), tf.compat.as_str(Constants.SAVED_MODEL_FILENAME_PB)); + File.WriteAllBytes(path, saved_model.ToByteArray()); + //File.WriteAllText(path, saved_model.ToString()); + + if (options.save_debug_info) + { + throw new NotImplementedException(); + } + + ops.dismantle_graph(exported_graph); + + return (saved_nodes, node_paths); + } + + private static (MetaGraphDef, Graph, TrackableSaver, AssetInfo, IList, + IDictionary>) _build_meta_graph(Trackable obj, + ConcreteFunction? signatures, SaveOptions options, MetaGraphDef? meta_graph_def = null) + { + using (SaveContext.save_context(options)) + { + if (ops.inside_function()) + { + throw new AssertionError("`tf.saved_model.save` is not supported inside a traced @tf.function. " + + "Move the call to the outer eagerly-executed context."); + } + + if (meta_graph_def is null) + { + meta_graph_def = new MetaGraphDef(); + } + + AugmentedGraphView augmented_graph_view = new AugmentedGraphView(obj); + if (signatures is null) + { + signatures = SignatureSerializationUtils.find_function_to_export(augmented_graph_view); + } + + // TODO: process of aignatures and wrapped_functions + + SaveableView saveable_view = new SaveableView(augmented_graph_view, options); + TrackableSaver object_saver = new TrackableSaver(augmented_graph_view); + var (asset_info, exported_graph) = _fill_meta_graph_def(meta_graph_def, saveable_view, signatures, + options.namespace_white_list, options.experimental_custom_gradients); + if (options.function_aliases is not null) + { + var function_aliases = meta_graph_def.MetaInfoDef.FunctionAliases; + foreach (var pair in options.function_aliases) + { + var alias = pair.Key; + var func = pair.Value; + // TODO: complete it. + throw new NotImplementedException(); + } + } + + var object_graph_proto = saveable_view.serialize_object_graph(asset_info.asset_index); + meta_graph_def.ObjectGraphDef = new SavedObjectGraph(object_graph_proto); + + return (meta_graph_def, exported_graph, object_saver, asset_info, saveable_view.Nodes, saveable_view.NodePaths); + } + } + + private static (AssetInfo, Graph) _fill_meta_graph_def(MetaGraphDef meta_graph_def, SaveableView saveable_view, + ConcreteFunction signatures, IEnumerable namespace_whitelist, + bool save_custom_gradients) + { + var resource_initializers = saveable_view.get_concrete_resource_initializers(); + var exported_graph = new Graph(); + + Dictionary object_map; + Dictionary tensor_map; + AssetInfo asset_info; + var g = exported_graph.as_default(); + (object_map, tensor_map, asset_info) = saveable_view.map_resources(); + // TODO: deal with signatures. + if (save_custom_gradients) + { + // TODO: trace gradient functions. + } + + foreach (var resource_initializer_function in resource_initializers) + { + // List asset_dependencies = new(); + // TODO: deal with initializers + } + + // using(ops.control_dependencies(...)) + var init_op = control_flow_ops.no_op(); + if (meta_graph_def.CollectionDef.ContainsKey(Tensorflow.Constants.MAIN_OP_KEY)) + { + meta_graph_def.CollectionDef[Tensorflow.Constants.MAIN_OP_KEY].NodeList.Value.Append(init_op.name); + } + else + { + meta_graph_def.CollectionDef[Tensorflow.Constants.MAIN_OP_KEY] = new CollectionDef(); + } + // Lack `CopyFrom` API + // meta_graph_def.SignatureDef[Tensorflow.Constants.INIT_OP_SIGNATURE_KEY] + + g.Exit(); + + foreach (var obj in object_map.Values) + { + obj._maybe_initialize_trackable(); + } + + // TODO: add the implementation of `call_with_mapped_functions`. + var (named_saveable_objects, registered_savers) = + SaveUtilV1.frozen_saveables_and_savers(saveable_view.AugmentedGraphView, object_map, exported_graph, false); + var saver = MultiDeviceSaver.from_saveables(named_saveable_objects, registered_savers, false); + + var eg = exported_graph.as_default(); + var saver_def = saver.to_proto(); + meta_graph_def.SaverDef = saver_def; + eg.Exit(); + + + saveable_view.dependency_sorted_node_ids(); + + var graph_def = exported_graph.as_graph_def(true); + graph_def.Library.RegisteredGradients.AddRange(saveable_view.GradientDefs); + verify_ops(graph_def, namespace_whitelist); + + meta_graph_def.GraphDef = new GraphDef(graph_def); + meta_graph_def.MetaInfoDef = new(); + meta_graph_def.MetaInfoDef.Tags.Add(TagConstants.SERVING); + meta_graph_def.MetaInfoDef.TensorflowVersion = tf.VERSION; + // TODO: add git version. + meta_graph_def.MetaInfoDef.TensorflowGitVersion = ""; + meta_graph_def.MetaInfoDef.StrippedDefaultAttrs = true; + meta_graph_def.MetaInfoDef.StrippedOpList = new(); + meta_graph_def.MetaInfoDef.StrippedOpList.MergeFrom(meta_graph.stripped_op_list_for_graph(meta_graph_def.GraphDef)); + meta_graph_def.AssetFileDef.AddRange(asset_info.asset_defs); + + // TODO: deal with signatures here. + + meta_graph.strip_graph_default_valued_attrs(meta_graph_def); + + if (!BitConverter.IsLittleEndian) + { + swap_function_tensor_content(meta_graph_def); + } + + return (asset_info, exported_graph); + } + + private static void verify_ops(GraphDef graph_def, IEnumerable? namespace_whitelist) + { + return; + // if (namespace_whitelist is null || !namespace_whitelist.Any()) + // { + // return; + // } + + // skip the check for the lack of `meta_graph.ops_used_by_graph_def`. + } + + public static void swap_function_tensor_content(MetaGraphDef meta_graph_def) + { + var functions = meta_graph_def.GraphDef.Library.Function; + foreach (var function in functions) + { + var node_def = function.NodeDef; + foreach (var node in node_def) + { + if (node.Op == "Const") + { + var tensor = node.Attr["value"].Tensor; + byte_swap_tensor_content(tensor); + } + } + } + } + + public static void byte_swap_tensor_content(TensorProto tensor) + { + if (byte_swappable.Contains((int)tensor.Dtype)) + { + var tshape = tensor.TensorShape.Dim; + var tensor_bytes = tensor.TensorContent; + if (tensor_bytes is not null && !tensor_bytes.IsEmpty) + { + long tensor_size = 1; + foreach (var sz in tshape) + { + tensor_size *= sz.Size; + } + + var chunksize = tensor_bytes.Length / tensor_size; + List reversed_bytes = new(); + for (int i = 0; i < tensor_bytes.Length; i += (int)chunksize) + { + var current = tensor_bytes.Skip(i).Take((int)chunksize).Reverse(); + reversed_bytes.AddRange(current); + } + tensor.TensorContent = ByteString.CopyFrom(reversed_bytes.ToArray()); + } + } + } +} diff --git a/src/TensorFlowNET.Core/Training/Saving/SavedModel/save_context.cs b/src/TensorFlowNET.Core/Training/Saving/SavedModel/save_context.cs new file mode 100644 index 000000000..4cfe0b69b --- /dev/null +++ b/src/TensorFlowNET.Core/Training/Saving/SavedModel/save_context.cs @@ -0,0 +1,53 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.ModelSaving; + +namespace Tensorflow.Training.Saving.SavedModel +{ + /// + /// A context for building a graph of SavedModel. + /// + public static class SaveContext + { + // TODO: make it thead safe. + private static bool _in_save_context = false; + private static SaveOptions _save_options = null; + + public static bool in_save_context() => _in_save_context; + public static SaveOptions get_save_options() + { + if (!in_save_context()) + { + throw new ValueError("Not in a SaveContext."); + } + return _save_options; + } + public static SaveContextHandler save_context(SaveOptions options) + { + return new SaveContextHandler(options); + } + + public class SaveContextHandler: IDisposable + { + private bool _old_in_save_context; + private SaveOptions _old_save_options; + public SaveContextHandler(SaveOptions options) + { + if (SaveContext.in_save_context()) + { + throw new ValueError("Already in a SaveContext."); + } + _old_in_save_context = SaveContext._in_save_context; + SaveContext._in_save_context = true; + _old_save_options = SaveContext._save_options; + SaveContext._save_options = options; + } + public void Dispose() + { + SaveContext._in_save_context = _old_in_save_context; + SaveContext._save_options = _old_save_options; + } + } + } +} diff --git a/src/TensorFlowNET.Core/Training/Saving/SavedModel/signature_serialization.cs b/src/TensorFlowNET.Core/Training/Saving/SavedModel/signature_serialization.cs new file mode 100644 index 000000000..4a0d3b002 --- /dev/null +++ b/src/TensorFlowNET.Core/Training/Saving/SavedModel/signature_serialization.cs @@ -0,0 +1,107 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Tensorflow.Functions; +using Tensorflow.Keras.Saving.SavedModel; +using Tensorflow.Train; + +namespace Tensorflow; + +public static class SignatureSerializationUtils +{ + internal static readonly string DEFAULT_SIGNATURE_ATTR = "_default_save_signature"; + internal static readonly string SIGNATURE_ATTRIBUTE_NAME = "signatures"; + internal static readonly int _NUM_DISPLAY_NORMALIZED_SIGNATURES = 5; + public static SignatureMap create_signature_map(IDictionary signatures) + { + var signature_map = new SignatureMap(); + foreach (var pair in signatures) + { + var name = pair.Key; + var func = pair.Value; + Debug.Assert(func is ConcreteFunction); + // TODO: assert the `func.structured_outputs` and arg_keywords. + signature_map._add_signature(name, (ConcreteFunction)func); + } + + return signature_map; + } + + public static ConcreteFunction find_function_to_export(AugmentedGraphView graph_view) + { + var children = graph_view.list_children(graph_view.Root); + List possible_signatures = new(); + foreach (var item in children) + { + var name = item.Name; + var child = item.Refer; + if(child is not (Function or ConcreteFunction)) + { + continue; + } + if(name == DEFAULT_SIGNATURE_ATTR) + { + Debug.Assert(child is ConcreteFunction); + return (ConcreteFunction)child; + } + ConcreteFunction concrete = get_signature(child); + if(concrete is not null && valid_signature(concrete)) + { + possible_signatures.Add(concrete); + } + } + + if(possible_signatures.Count == 1) + { + var signature = get_signature(possible_signatures[0]); + if(signature is not null && valid_signature(signature)) + { + return signature; + } + } + return null; + } + + private static ConcreteFunction get_signature(Trackable function) + { + // TODO: implement it. + return null; + } + + private static bool valid_signature(ConcreteFunction concreate_function) + { + // TODO: implement it. + return false; + } +} + +public class SignatureMap: Trackable +{ + private Dictionary _signatures; + + public SignatureMap() + { + _signatures = new(); + } + + public void _add_signature(string name, ConcreteFunction concrete_function) + { + _signatures[name] = concrete_function; + } + + public void _add_signature(string name, Function concrete_function) + { + _signatures[name] = concrete_function; + } + + public override IDictionary _trackable_children(SaveType save_type, IDictionary>? cache = null) + { + if (save_type != SaveType.SAVEDMODEL) + { + return new Dictionary(); + } + + return _signatures.TakeWhile(x => x.Value is Function or ConcreteFunction).ToDictionary(x => x.Key, x => x.Value); + } +} diff --git a/src/TensorFlowNET.Core/Training/Saving/SavedModel/utils.cs b/src/TensorFlowNET.Core/Training/Saving/SavedModel/utils.cs new file mode 100644 index 000000000..b0e6411c9 --- /dev/null +++ b/src/TensorFlowNET.Core/Training/Saving/SavedModel/utils.cs @@ -0,0 +1,57 @@ +using System.IO; +using System.Security.Cryptography.X509Certificates; +using Tensorflow.Train; +using static Tensorflow.Binding; + +namespace Tensorflow; + +public static partial class SavedModelUtils +{ + /// + /// Return variables sub-directory, or create one if it doesn't exist. + /// + /// + public static string get_or_create_variables_dir(string export_dir) + { + var variables_dir = get_variables_dir(export_dir); + Directory.CreateDirectory(variables_dir); + return variables_dir; + } + + /// + /// Return variables sub-directory in the SavedModel. + /// + /// + /// + public static string get_variables_dir(string export_dir) + { + return Path.Combine(tf.compat.as_text(export_dir), tf.compat.as_text(Constants.VARIABLES_DIRECTORY)); + } + + public static string get_variables_path(string export_dir) + { + return Path.Combine(tf.compat.as_text(get_variables_dir(export_dir)), tf.compat.as_text(Constants.VARIABLES_FILENAME)); + } + + /// + /// Return assets sub-directory, or create one if it doesn't exist. + /// + /// + /// + public static string get_or_create_assets_dir(string export_dir) + { + var assets_destination_dir = get_assets_dir(export_dir); + Directory.CreateDirectory(assets_destination_dir); + return assets_destination_dir; + } + + /// + /// Return path to asset directory in the SavedModel. + /// + /// + /// + public static string get_assets_dir(string export_dir) + { + return Path.Combine(tf.compat.as_text(export_dir), tf.compat.as_text(Constants.ASSETS_DIRECTORY)); + } +} diff --git a/src/TensorFlowNET.Core/Training/Saving/saveable_object_util.py.cs b/src/TensorFlowNET.Core/Training/Saving/saveable_object_util.py.cs index 3a6647880..a6e21e3e5 100644 --- a/src/TensorFlowNET.Core/Training/Saving/saveable_object_util.py.cs +++ b/src/TensorFlowNET.Core/Training/Saving/saveable_object_util.py.cs @@ -16,12 +16,38 @@ limitations under the License. using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; +using Tensorflow.Checkpoint; +using Tensorflow.Operations.Activation; +using Tensorflow.Train; +using Tensorflow.Training; using static Tensorflow.Binding; namespace Tensorflow { - public class saveable_object_util + /// + /// A SaveableObject that defines `Trackable` checkpointing steps. + /// + public class TrackableSaveable : MySaveableObject + { + private string _prefix; + private IEnumerable _local_names; + private Trackable _trackable; + private bool _call_with_mapped_captures; + // TODO: revise the implementation. Currently the parameter of constructor of this class and its base class has conflict. + public TrackableSaveable(Trackable obj, IEnumerable specs, string name, IEnumerable local_names, + string prefix, bool call_with_mapped_captures = false) : base((object)obj as Tensor, specs.ToArray(), name) + { + _prefix = prefix; + _trackable = obj; + _local_names = local_names; + _call_with_mapped_captures = call_with_mapped_captures; + } + + // TODO: complete this class. + } + public static class saveable_object_util { /// /// Returns the variables and names that will be used for a Saver. @@ -52,7 +78,7 @@ private static void _add_saveable(List saveables, List seen_ops, T } /// - /// Create `SaveableObject`s from an operation. + /// Create `SaveableObject`s from an operation. Note that the `op` should not be implicitly converted from `Variable`. /// /// /// @@ -74,6 +100,72 @@ public static IEnumerable saveable_objects_for_op(Tensor op, s } } + /// + /// Create `SaveableObject`s from an operation. + /// + /// + /// + /// + public static IEnumerable saveable_objects_for_op(Trackable obj, string name) + { + // The `op` maybe `Variable` or `Trackable`. + if (obj is BaseResourceVariable) + { + var variable = obj as BaseResourceVariable; + if (variable.InGraphMode) + { + yield return new ResourceVariableSaveable(variable.GraphElement, "", name); + } + else + { + yield return new ResourceVariableSaveable(variable, "", name); + } + } + else + { + foreach(var pair in saveable_objects_from_trackable(obj)) + { + var attr = pair.Key; + var factory = pair.Value; + string full_name; + if(attr == Trackable.Constants.VARIABLE_VALUE_KEY) + { + full_name = name; + } + else + { + full_name = name + "_" + attr; + } + if(factory.TryGet(out var variable)) + { + foreach (var op in saveable_objects_for_op(variable as Trackable, variable.Name)) + { + yield return op; + } + } + else + { + var saveable = factory.GetValue(); + foreach (var op in saveable_objects_for_op(saveable, saveable.name)) + { + yield return op; + } + } + } + } + } + + /// + /// Create `SaveableObject`s from an operation. + /// + /// + /// + /// + public static IEnumerable saveable_objects_for_op(MySaveableObject obj, string name) + { + yield return obj; + } + public static Dictionary op_list_to_dict(IVariableV1[] op_list, bool convert_variable_to_tensor = true) { op_list = op_list.OrderBy(x => x.Name).ToArray(); @@ -121,5 +213,164 @@ public static Dictionary op_list_to_dict(IVariableV1[] op_list, return names_to_saveables; } + + public static IDictionary> saveable_objects_from_trackable(Trackable obj) + { + // skip the process of type `PythonState` + + if (trackable_has_serialize_to_tensor(obj)) + { + var name = TrackableUtils.SERIALIZE_TO_TENSORS_NAME; + // skip the case that `obj._serialize_to_tensors` is `ConcreteFunction`. + var tensor_dict = obj.serialize_to_tensors(); + + List specs = new(); + List local_names = new(); + string prefix = SaveableCompat.get_saveable_name(obj) ?? ""; + foreach(var pair in tensor_dict) + { + var tensor_name = pair.Key; + var maybe_tensor = pair.Value; + local_names.Add(tensor_name); + string spec_name = name + TrackableUtils.escape_local_name(tensor_name); + + IDictionary internal_dict; + if(maybe_tensor.TryGet(out var tensor)) + { + internal_dict= new Dictionary(); + internal_dict[""] = tensor; + } + else + { + internal_dict = maybe_tensor.GetValue>(); + } + + foreach(var item in internal_dict) + { + specs.Add(new SaveSpec(item.Value, item.Key, spec_name)); + } + } + Dictionary> res = new(); + res[name] = new TrackableSaveable(obj, specs, name, local_names, prefix); + return res; + } + else + { + return obj.gather_saveables_for_checkpoint(); + } + } + + public static bool trackable_has_serialize_to_tensor(Trackable obj) + { + return obj.GetType().GetMethod("serialize_to_tensors").DeclaringType != typeof(Trackable); + } + + internal static string convert_to_string(string x) + { + return tf.compat.as_str(x); + } + + /// + /// Converts a list of SaveableObjects to a tensor dictionary. + /// + /// + public static Dictionary>> saveable_object_to_tensor_dict(IList saveables) + { + Dictionary>> tensor_dict = new(); + foreach (var saveable in saveables) + { + foreach (var spec in saveable.specs) + { + // skip the check that if `spec` is callable. + var name = convert_to_string(spec.name); + var slice_spec = convert_to_string(spec.slice_spec); + if (!string.IsNullOrEmpty(slice_spec)) + { + tensor_dict.SetDefault(name, new Dictionary()).GetValue>()[slice_spec] = spec.tensor; + } + else + { + tensor_dict[name] = spec.tensor; + } + } + } + return tensor_dict; + } + + /// + /// Generates `Trackable._restore_from_tensors` from SaveableObjects. + /// + /// + public static Func>>, IDictionary> saveable_object_to_restore_fn(IList saveables) + { + return (restored_tensors) => + { + Dictionary restored_ops = new(); + + foreach(var saveable in saveables) + { + List saveable_restored_tensors = new(); + foreach(var spec in saveable.specs) + { + var name = TrackableUtils.extract_local_name(saveable_object_util.convert_to_string(spec.name)); + var slice_spec = saveable_object_util.convert_to_string(spec.slice_spec); + + var maybe_tensor = restored_tensors[name]; + IDictionary dict; + if(maybe_tensor.TryGet(out var tensor)) + { + dict = new Dictionary(); + dict[""] = tensor; + } + else + { + dict = maybe_tensor.GetValue>(); + } + saveable_restored_tensors.Add(dict[slice_spec]); + } + restored_ops[saveable.name] = saveable.restore(saveable_restored_tensors.ToArray(), null); + } + return restored_ops; + }; + } + } + + public class SaveableCompatibilityConverter: Trackable + { + private object _obj; + private IList _saveables; + public SaveableCompatibilityConverter(object obj, IList saveables) + { + _obj= obj; + _saveables= saveables; + } + + public object Obj => _obj; + public IList mySaveables=> _saveables; + + public override IDictionary>> serialize_to_tensors() + { + return saveable_object_util.saveable_object_to_tensor_dict(_saveables); + } + + /// + /// Returns the restore ops defined in the Saveables. + /// + /// + /// + public override IDictionary _restore_from_tensors(IDictionary>> restored_tensors) + { + List expected_keys = new(); + foreach(var saveable in _saveables) + { + expected_keys.AddRange(saveable.specs.Select(x => TrackableUtils.extract_local_name(saveable_object_util.convert_to_string(x.name)))); + } + if (!expected_keys.Distinct().SequenceEqual(restored_tensors.Keys)) + { + throw new ValueError($"Could not restore object {_obj} because not all expected tensors were in the checkpoint." + + $"\n\tExpected: {expected_keys} \n\tGot: {list(restored_tensors.Keys)}"); + } + return saveable_object_util.saveable_object_to_restore_fn(_saveables).Invoke(restored_tensors); + } } } diff --git a/src/TensorFlowNET.Core/Training/Trackable.cs b/src/TensorFlowNET.Core/Training/Trackable.cs index 79d6dca92..132571f2a 100644 --- a/src/TensorFlowNET.Core/Training/Trackable.cs +++ b/src/TensorFlowNET.Core/Training/Trackable.cs @@ -14,13 +14,63 @@ You may obtain a copy of the License at limitations under the License. ******************************************************************************/ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Tensorflow.Checkpoint; +using Tensorflow.Keras.Saving.SavedModel; +using Tensorflow.ModelSaving; +using Tensorflow.Training; using static Tensorflow.Binding; namespace Tensorflow.Train { - public abstract class Trackable + public abstract class Trackable: IWithTrackable { + /// + /// Corresponding to tensorflow/python/trackable/constants.py + /// + public static class Constants + { + public static readonly string OBJECT_GRAPH_PROTO_KEY = "_CHECKPOINTABLE_OBJECT_GRAPH"; + public static readonly string VARIABLE_VALUE_KEY = "VARIABLE_VALUE"; + public static readonly string OBJECT_CONFIG_JSON_KEY = "OBJECT_CONFIG_JSON"; + } protected int _self_update_uid; + protected IDictionary _unconditional_dependency_names; + + protected IList _unconditional_checkpoint_dependencies; + + protected IDictionary> _self_saveable_object_factories = + new Dictionary>(); + private bool _manual_tracking = true; + + private static Trackable _none = new AutoTrackable(); + /// + /// This is a trick for that CSharp does not allow the key of `Dictionary` to be null. + /// The `None` can be any object that inherits `Trackable`. + /// This Property is supposed to be used only internal. + /// + public static Trackable None + { + get + { + return _none; + } + } + public Trackable GetTrackable() + { + return this; + } + public virtual string ObjectIdentifier + { + get => "_generic_user_object"; + } + public int UpdateUid { get => _self_update_uid; set => _self_update_uid = value; } + public IList UnconditionalCheckpointDependencies { get => _unconditional_checkpoint_dependencies; } + public IDictionary UnconditionalDependencyNames { get => _unconditional_dependency_names; } + public IList CheckpointDependencies { get => UnconditionalCheckpointDependencies; } /// /// Restore-on-create for a variable be saved with this `Checkpointable`. @@ -47,9 +97,13 @@ protected virtual IVariableV1 _add_variable_with_custom_getter(VariableArgs args // assign again. It will add this variable to our dependencies, and if there // is a non-trivial restoration queued, it will handle that. This also // handles slot variables. - if (!args.Overwrite || new_variable is RefVariable) - return _track_checkpointable(new_variable, name: args.Name, - overwrite: args.Overwrite); + if (!args.Overwrite || new_variable is RefVariable || new_variable is Trackable) + { + var temp = new_variable as Trackable; + var res = _track_trackable(temp, args.Name, args.Overwrite); + Debug.Assert(res is IVariableV1); + return res as IVariableV1; + } else return new_variable; } @@ -73,10 +127,136 @@ protected IVariableV1 _track_checkpointable(IVariableV1 checkpointable, string n /// /// Initialize dependency management. /// - protected void _maybe_initialize_trackable() + public void _maybe_initialize_trackable() { - // _self_unconditional_checkpoint_dependencies = [] + if(_unconditional_checkpoint_dependencies is not null) + { + return; + } _self_update_uid = -1; + _unconditional_checkpoint_dependencies = new List(); + _unconditional_dependency_names = new Dictionary(); + } + + public virtual IDictionary _trackable_children(SaveType save_type, IDictionary>? cache) + { + _maybe_initialize_trackable(); + return _unconditional_checkpoint_dependencies.ToDictionary(x => x.Name, x => x.Refer); + } + + public virtual Trackable _track_trackable(Trackable trackable, string name, bool overwrite = false) + { + _maybe_initialize_trackable(); + if (!_manual_tracking) return trackable; + var new_reference = new TrackableReference(name, trackable); + var current_object = _lookup_dependency(name); + + if(current_object is null) + { + _unconditional_checkpoint_dependencies.Add(new_reference); + _handle_deferred_dependencies(name, trackable); + } + _unconditional_dependency_names[name] = trackable; + return trackable; + } + + /// + /// Pop and load any deferred checkpoint restores into `trackable`. + /// This method does not add a new dependency on `trackable`, but it does check if any outstanding/deferred dependencies have been queued waiting for + /// this dependency to be added (matched based on `name`). If so, `trackable` and its dependencies are restored. The restorations are + /// considered fulfilled and so are deleted. + /// `_track_trackable` is more appropriate for adding a normal/unconditional dependency, and includes handling for deferred restorations. + /// This method allows objects such as `Optimizer` to use the same restoration logic while managing conditional dependencies themselves, + /// by overriding `_checkpoint_dependencies` and `_lookup_dependency` to change the object's dependencies based on the context + /// it is saved/restored in (a single optimizer instance can have state associated with multiple graphs). + /// + /// + /// + public virtual void _handle_deferred_dependencies(string name, Trackable trackable) + { + //_maybe_initialize_trackable(); + //trackable._maybe_initialize_trackable(); + + // TODO: complete the implementation. + } + + public virtual Trackable? _lookup_dependency(string name) + { + if (_unconditional_dependency_names.TryGetValue(name, out var dependency)) return dependency; + else return null; + } + + public static Trackable convert_to_trackable(object obj, object? parent = null) + { + if (obj is Trackable) + { + return (Trackable)obj; + } + else + { + throw new NotImplementedException(); + } + } + + public virtual IDictionary deserialization_dependencies(IDictionary children) + { + return new Dictionary(); + } + + public virtual (IDictionary, IDictionary) map_resources( + SaveOptions? save_options) + { + return (new Dictionary(), new Dictionary()); + } + + public virtual List export_to_saved_model_graph(IDictionary object_map, + IDictionary tensor_map, SaveOptions? options = null) + { + var (self_object_map, self_tensor_map) = map_resources(options); + foreach (var pair in self_object_map) + { + object_map.Add(pair); + } + foreach (var pair in self_tensor_map) + { + tensor_map.Add(pair); + } + + return self_tensor_map.Keys.ToList(); + } + + public virtual IDictionary> gather_saveables_for_checkpoint() + { + if (saveable_object_util.trackable_has_serialize_to_tensor(this)) + { + // TODO: complete the implementation (need to complete the class `saveable_object_util.TrackableSaveable`). + throw new NotImplementedException(); + } + else + { + return _self_saveable_object_factories; + } + } + + /// + /// Gathers tensors to save to the checkpoint. You should only override `serialize_to_tensors` and `restore_from_tensors` + /// if you are defining a custom resource or variable with custom ops. + /// Otherwise, please store the state of your trackable in `tf.Variable` objects + /// and add them to Trackable object hierarchy using `setattr` (for subclasses + /// of `AutoTrackable`) or overriding the `_trackable_children` method. + /// + /// + /// + public virtual IDictionary>> serialize_to_tensors() + { + throw new NotImplementedException(); + } + + public virtual IDictionary _restore_from_tensors(IDictionary>> restored_tensors) + { + throw new NotImplementedException(); } } + + public record class TrackableReference(string Name, Trackable Refer); } diff --git a/src/TensorFlowNET.Core/Training/TrackableUtils.cs b/src/TensorFlowNET.Core/Training/TrackableUtils.cs new file mode 100644 index 000000000..390d95c75 --- /dev/null +++ b/src/TensorFlowNET.Core/Training/TrackableUtils.cs @@ -0,0 +1,172 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Tensorflow.Exceptions; +using Tensorflow.Train; + +namespace Tensorflow.Training; + +public static class TrackableUtils +{ + public class CyclicDependencyError: System.Exception + { + public IDictionary> LeftOverDependencyMap { get; } + public CyclicDependencyError(IDictionary> leftover_dependency_map): base() + { + LeftOverDependencyMap = leftover_dependency_map; + } + public CyclicDependencyError(IDictionary> leftover_dependency_map): base() + { + LeftOverDependencyMap = leftover_dependency_map.ToDictionary(x => x.Key, x => x.Value.AsEnumerable()); + } + } + private static string _ESCAPE_CHAR = "."; + private static string _OPTIMIZER_SLOTS_NAME = _ESCAPE_CHAR + "OPTIMIZER_SLOT"; + private static string OBJECT_ATTRIBUTES_NAME = _ESCAPE_CHAR + "ATTRIBUTES"; + internal static string SERIALIZE_TO_TENSORS_NAME = _ESCAPE_CHAR + "TENSORS"; + public static string object_path_to_string(IEnumerable node_path_arr) + { + return string.Join("/", node_path_arr.Select(x => escape_local_name(x.Name))); + } + + public static string escape_local_name(string name) + { + return name.Replace(_ESCAPE_CHAR, _ESCAPE_CHAR + _ESCAPE_CHAR).Replace("/", _ESCAPE_CHAR + "S"); + } + + public static string checkpoint_key(string object_path, string local_name) + { + var key_suffix = escape_local_name(local_name); + if (local_name == SERIALIZE_TO_TENSORS_NAME) + { + key_suffix = ""; + } + + return $"{object_path}/{OBJECT_ATTRIBUTES_NAME}/{key_suffix}"; + } + + /// + /// Topologically sorts the keys of a map so that dependencies appear first. + /// Uses Kahn's algorithm: https://en.wikipedia.org/wiki/Topological_sorting#Kahn's_algorithm + /// + /// + /// + public static List order_by_dependency(IDictionary> dependency_map) + { + Dictionary> reverse_dependency_map = new(); + foreach (var pair in dependency_map) + { + foreach (var dep in pair.Value) + { + if (reverse_dependency_map.ContainsKey(dep)) + { + reverse_dependency_map[dep].Add(pair.Key); + } + else + { + reverse_dependency_map[dep] = new HashSet(); + reverse_dependency_map[dep].Add(pair.Key); + } + } + } + + // Validate that all values in the dependency map are also keys. + var unknown_keys = reverse_dependency_map.Keys.Except(dependency_map.Keys); + if (unknown_keys.Count() > 0) + { + throw new ValueError( + $"Found values in the dependency map which are not keys: {string.Join(", ", unknown_keys.Select(x => x.ToString()))}"); + } + + // Generate the list sorted by objects without dependencies -> dependencies. + // The returned list will reverse this. + List reversed_dependency_arr = new(); + + Queue to_visit = new(); + foreach (var x in dependency_map.Keys) + { + if (!reverse_dependency_map.ContainsKey(x)) + { + to_visit.Enqueue(x); + } + } + + while (to_visit.Count > 0) + { + var x = to_visit.Dequeue(); + reversed_dependency_arr.Add(x); + foreach (var dep in dependency_map[x].Distinct()) + { + var edges = reverse_dependency_map[dep]; + edges.Remove(x); + if (edges.Count == 0) + { + to_visit.Enqueue(dep); + if (!reverse_dependency_map.Remove(dep)) + { + throw new KeyError($"Cannot find the key {dep} in reverse_dependency_map"); + } + } + } + } + + if (reverse_dependency_map.Count > 0) + { + Dictionary> leftover_dependency_map = new(); + foreach (var pair in reverse_dependency_map) + { + foreach (var x in pair.Value) + { + if (leftover_dependency_map.ContainsKey(x)) + { + leftover_dependency_map[x].Add(pair.Key); + } + else + { + leftover_dependency_map[x] = new List() { pair.Key }; + } + } + } + + throw new CyclicDependencyError(leftover_dependency_map); + } + + reversed_dependency_arr.Reverse(); + return reversed_dependency_arr; + } + + public static string pretty_print_node_path(IEnumerable paths) + { + if (paths.Count() == 0) + { + return "root object"; + } + else + { + return $"root.{string.Join(".", paths.Select(x => x.Name))}"; + } + } + + /// + /// Returns the substring after the "/.ATTIBUTES/" in the checkpoint key. + /// + /// + /// + /// + public static string extract_local_name(string key, string? prefix = null) + { + if(prefix is null) + { + prefix = ""; + } + var search_key = OBJECT_ATTRIBUTES_NAME + "/" + prefix; + try + { + return key.Substring(key.IndexOf(search_key) + search_key.Length); + } + catch(ArgumentOutOfRangeException) + { + return key; + } + } +} \ No newline at end of file diff --git a/src/TensorFlowNET.Core/Training/data_structures.cs b/src/TensorFlowNET.Core/Training/data_structures.cs new file mode 100644 index 000000000..6e3336c90 --- /dev/null +++ b/src/TensorFlowNET.Core/Training/data_structures.cs @@ -0,0 +1,370 @@ +using Google.Protobuf; +using System; +using System.Collections; +using System.Collections.Generic; +using System.IO.Compression; +using System.Linq; +using System.Linq.Expressions; +using System.Runtime.InteropServices; +using System.Text; +using Tensorflow.Functions; +using Tensorflow.Keras; +using Tensorflow.Keras.Saving.SavedModel; +using Tensorflow.Operations.Activation; +using Tensorflow.Train; +using static Tensorflow.ApiDef.Types; + +namespace Tensorflow.Training +{ + public class NoDependency + { + public Trackable Value { get; set; } + public NoDependency(Trackable value) + { + Value = value; + } + } + + public abstract class TrackableDataStructure : Trackable + { + private bool _self_trainable; + private List _self_extra_variables; + + public TrackableDataStructure() + { + _self_trainable = true; + _self_extra_variables = new List(); + } + + public abstract IEnumerable Values { get; } + public bool Trainable { get => _self_trainable; set => _self_trainable = value; } + public IEnumerable Layers + { + get + { + List collected = new(); + foreach(var obj in Values) + { + if(obj is ILayer) + { + collected.Add((ILayer)obj); + } + else if(obj is TrackableDataStructure) + { + collected.AddRange((obj as TrackableDataStructure).Layers); + } + } + return collected; + } + } + public IEnumerable TrainableWeights + { + get + { + if (!_self_trainable) + { + return new List(); + } + List trainable_variables = new(); + foreach (var obj in Values) + { + // skip the process of `module.Module`. + if (obj is TrackableDataStructure) + { + trainable_variables.AddRange((obj as TrackableDataStructure).TrainableVariables); + } + } + foreach(var v in _self_extra_variables) + { + if (v.Trainable) + { + trainable_variables.Add(v); + } + } + return trainable_variables; + } + } + public IEnumerable NonTrainableWeights + { + get + { + var trainable_extra_variables = _self_extra_variables.TakeWhile(x => x.Trainable).ToList(); + var non_trainable_extra_variables = _self_extra_variables.TakeWhile(x => !x.Trainable).ToList(); + List non_trainable_variables = new(); + foreach(var obj in Values) + { + // skip the process of `module.Module`. + if (obj is TrackableDataStructure) + { + non_trainable_variables.AddRange((obj as TrackableDataStructure).NonTrainableVariables); + } + } + + if (!_self_trainable) + { + // Return order is all trainable vars, then all non-trainable vars. + List trainable_variables = new(); + foreach(var obj in Values) + { + // skip the process of `module.Module`. + if (obj is TrackableDataStructure) + { + trainable_variables.AddRange((obj as TrackableDataStructure).TrainableVariables); + } + } + return trainable_variables.concat(trainable_extra_variables).concat(non_trainable_variables).concat(non_trainable_extra_variables); + } + else + { + return non_trainable_variables.concat(non_trainable_extra_variables); + } + } + } + public IEnumerable Weights => TrainableWeights.Concat(NonTrainableWeights); + public IEnumerable TrainableVariables => TrainableWeights; + public IEnumerable NonTrainableVariables => NonTrainableWeights; + public IEnumerable Variables => Weights; + + // TODO: `losses` property. + + /// + /// Add a dependency on `value`. + /// + /// + /// + protected virtual Trackable _track_value(Trackable value, string name) + { + value = sticky_attribute_assignment(this, name, value); + if(value is IVariableV1) + { + _self_extra_variables.Add(value as IVariableV1); + } + // skip the left process (need to be done in the future). + return value; + } + + public static Trackable wrap_or_unwrap(NoDependency value) + { + return value.Value; + } + + public static Trackable wrap_or_unwrap(Trackable value) + { + return value; + } + + public static Trackable wrap_or_unwrap(IList value) + { + return new ListWrapper(value); + } + + public static Trackable wrap_or_unwrap(IEnumerable value) + { + return new ListWrapper(value.ToList()); + } + + protected static Trackable sticky_attribute_assignment(Trackable trackable, string name, Trackable value) + { + value = wrap_or_unwrap(value); + trackable._track_trackable(value, name, true); + return value; + } + + protected static Trackable sticky_attribute_assignment(Trackable trackable, string name, NoDependency value) + { + var wrapped_value = wrap_or_unwrap(value); + trackable._track_trackable(wrapped_value, name, true); + return wrapped_value; + } + + protected static Trackable sticky_attribute_assignment(Trackable trackable, string name, IList value) + { + var wrapped_value = wrap_or_unwrap(value); + trackable._track_trackable(wrapped_value, name, true); + return wrapped_value; + } + } + + public class ListWrapper : TrackableDataStructure, IList, ICloneable + { + private IList _storage; + private bool _non_append_mutation_value; + private bool _external_modification_value; + private IList _last_wrapped_list_snapshot; + /// + /// + /// + /// The initial value of the data structure. A shallow copy may be maintained for error checking. `wrapped_list` itself should not be + /// modified directly after constructing the `ListWrapper`, and if changes are detected the `ListWrapper` will throw an exception on save. + public ListWrapper(IList wrapped_list) + { + _storage = wrapped_list; + _non_append_mutation_value = _external_modification_value = false; + _last_wrapped_list_snapshot = new List(_storage); + } + + protected bool NonAppendMuation { + get => _non_append_mutation_value; + set + { + // TODO: deal with `attribute_sentinel`. + _non_append_mutation_value = value; + } + } + + protected bool ExternalModification + { + get => _external_modification_value; + set + { + // TODO: deal with `attribute_sentinel`. + _external_modification_value = value; + } + } + + public override IEnumerable Values => this; + public bool IsReadOnly { get => _storage.IsReadOnly; } + + /// + /// Checks for any changes to the wrapped list not through the wrapper. + /// + private void check_external_modification() + { + if (_external_modification_value || _non_append_mutation_value) return; + if (!_storage.SequenceEqual(_last_wrapped_list_snapshot)) + { + _external_modification_value = true; + } + } + + private void update_snapshot() + { + // TODO: deal with `attribute_sentinel`. + if (_external_modification_value || _non_append_mutation_value) return; + _last_wrapped_list_snapshot = new List(_storage); + } + + public override IDictionary _trackable_children(SaveType save_type, IDictionary>? cache = null) + { + check_external_modification(); + if (_non_append_mutation_value) + { + throw new ValueError($"Unable to save the object {this} (a list wrapper constructed to track trackable TensorFlow objects). A list element was replaced" + + $", deleted or moved (sort). In order to support restoration on object creation, tracking is exclusively for append-only data structures." + + $"\n\nIf you don't need this list checkpointed, wrap it in a non-trackable object; it will be subsequently ignored."); + } + if (_external_modification_value) + { + throw new ValueError($"Unable to save the object {this} (a list wrapper constructed to track trackable TensorFlow objects). The wrapped list was modified " + + $"outside the wrapper (its final value was {_storage}, its value when a checkpoint dependency was added was {_last_wrapped_list_snapshot}), which breaks " + + $"restoration on object creation.\n\nIf you don't need this list checkpointed, wrap it in a NoDependency object; it will be subsequently ignored."); + } + var children = base._trackable_children(save_type, cache); + + if(save_type == SaveType.SAVEDMODEL) + { + children = children.Concat(this.TakeWhile(x => x is Function or ConcreteFunction).Select((x, idx) => new KeyValuePair(idx.ToString(), x))).ToDictionary(x => x.Key, x => x.Value); + } + + return children; + } + + private bool has_mutation_or_trackable() + { + return _non_append_mutation_value; + } + + /// + /// Allows storage of non-trackable objects. + /// + /// + /// + /// + protected override Trackable _track_value(Trackable value, string name) + { + try + { + base._track_value(value, name); + } + catch(ValueError ex) + { + value = sticky_attribute_assignment(this, name, value); + } + return value; + } + + public object Clone() + { + var res = new ListWrapper(_storage.Select(x => x).ToList()); + res.NonAppendMuation= _non_append_mutation_value; + res.ExternalModification = _external_modification_value; + return res; + } + + public Trackable this[int index] { + get => _storage[index]; + set + { + // skip the process of `Slice`, maybe support it in the future. + _non_append_mutation_value = true; + _storage[index] = _track_value(value, _name_element(index)); + + update_snapshot(); + } + } + + public int IndexOf(Trackable item) => _storage.IndexOf(item); + + public void Insert(int index, Trackable item) + { + check_external_modification(); + _non_append_mutation_value = true; + _storage.Insert(index, item); + update_snapshot(); + } + + public void RemoveAt(int index) + { + check_external_modification(); + if (has_mutation_or_trackable()) + { + _non_append_mutation_value = true; + } + _storage.RemoveAt(index); + update_snapshot(); + } + + public int Count { get => _storage.Count; } + + public void Add(Trackable item) + { + check_external_modification(); + _storage.Add(item); + update_snapshot(); + } + + public void Clear() => _storage.Clear(); + + public bool Contains(Trackable item) => _storage.Contains(item); + + public void CopyTo(Trackable[] array, int arrayIndex) => _storage.CopyTo(array, arrayIndex); + + public bool Remove(Trackable item) + { + check_external_modification(); + if (has_mutation_or_trackable()) + { + _non_append_mutation_value = true; + } + var res = _storage.Remove(item); + update_snapshot(); + return res; + } + + public IEnumerator GetEnumerator() => _storage.GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => _storage.GetEnumerator(); + + protected string _name_element(int index) => $"{index}"; + } +} diff --git a/src/TensorFlowNET.Core/Variables/BaseResourceVariable.cs b/src/TensorFlowNET.Core/Variables/BaseResourceVariable.cs index b270ec57d..4005d5640 100644 --- a/src/TensorFlowNET.Core/Variables/BaseResourceVariable.cs +++ b/src/TensorFlowNET.Core/Variables/BaseResourceVariable.cs @@ -2,14 +2,20 @@ using System; using Tensorflow.Eager; using Tensorflow.Variables; +using Tensorflow.Train; using static Tensorflow.Binding; +using System.Collections.Generic; +using Tensorflow.ModelSaving; +using System.Diagnostics; +using Tensorflow.Checkpoint; namespace Tensorflow { - public class BaseResourceVariable : DisposableObject + public class BaseResourceVariable : DisposableTrackableObject { protected string _name; public virtual string Name => _handle_name; + public virtual string SharedName => _name; protected TF_DataType _dtype; public TF_DataType dtype => _dtype; protected string _handle_name; @@ -19,9 +25,10 @@ public class BaseResourceVariable : DisposableObject public string UniqueId => _unique_id; protected bool _in_graph_mode; + internal bool InGraphMode => _in_graph_mode; protected bool _trainable; - public bool trainable => _trainable; + public bool Trainable => _trainable; protected Tensor _initial_value; @@ -46,6 +53,7 @@ public class BaseResourceVariable : DisposableObject public Graph Graph => handle.graph; public string Device => handle.Device; EagerResourceDeleter eager_resource_deleter; + public VariableAggregation Aggregation { get; protected set; } = VariableAggregation.None; public BaseResourceVariable() { @@ -73,6 +81,11 @@ public void __init__(bool trainable = true, _handle = handle.EagerTensorHandle.DangerousGetHandle(); eager_resource_deleter = new EagerResourceDeleter(handle, handle.Device); } + else if(handle is null) + { + // TODO: fix this dangerous change. + _handle = IntPtr.Zero; + } else { _handle = handle.Handle == null ? IntPtr.Zero : handle.Handle.DangerousGetHandle(); @@ -165,7 +178,7 @@ IVariableV1 _lazy_read(Operation op, Tensor value) /// void variable_accessed(BaseResourceVariable variable) { - if (variable.trainable) + if (variable.Trainable) { foreach (var tape in tf.GetTapeSet()) tape.VariableAccessed(variable as ResourceVariable); @@ -243,5 +256,60 @@ public Tensor AsTensor(TF_DataType dtype = TF_DataType.DtInvalid, string name = else return value(); } + + public override (IDictionary, IDictionary) map_resources(SaveOptions save_options) + { + BaseResourceVariable new_variable; + if (save_options.experimental_variable_policy.save_variable_devices()) + { + tf.device(this.Device); + Debug.Assert(this is ResourceVariable); + new_variable = resource_variable_ops.copy_to_graph_uninitialized((ResourceVariable)this); + } + else + { + new_variable = resource_variable_ops.copy_to_graph_uninitialized((ResourceVariable)this); + } + Dictionary obj_map = new(); + Dictionary resource_map = new(); + obj_map[this] = new_variable; + resource_map[this.handle] = new_variable.handle; + return (obj_map, resource_map); + } + + /// + /// Writes additional information of the variable into the SavedObject proto. + /// ubclasses of ResourceVariables could choose to override this method to + /// customize extra information to provide when saving a SavedModel. + /// + /// + /// + public virtual void write_object_proto(SavedObject proto, SaveOptions options) + { + resource_variable_ops.write_object_proto_for_resource_variable(this, proto, options); + } + + public override IDictionary> gather_saveables_for_checkpoint() + { + var res = new Dictionary>(); + res[Trackable.Constants.VARIABLE_VALUE_KEY] = this; + return res; + } + + public Tensor is_initialized(string name = null) + { + return gen_resource_variable_ops.var_is_initialized_op(this.handle, name); + } + + public Tensor read_value_no_copy() + { + Tensor value = null; + tf_with(ops.name_scope("Read"), _ => + { + // TODO: `no_copy = true`. + value = _read_variable_op(); + }); + return array_ops.identity(value); + } } } diff --git a/src/TensorFlowNET.Core/Variables/IVariableV1.cs b/src/TensorFlowNET.Core/Variables/IVariableV1.cs index f4f716c3c..3eb78153a 100644 --- a/src/TensorFlowNET.Core/Variables/IVariableV1.cs +++ b/src/TensorFlowNET.Core/Variables/IVariableV1.cs @@ -46,6 +46,7 @@ public interface IVariableV1 Graph Graph { get; } TF_DataType dtype { get; } Shape shape { get; } + bool Trainable { get; } Tensor assign_add(T delta, bool use_locking = false, string name = null, bool read_value = true); Tensor assign_sub(T delta, bool use_locking = false, string name = null, bool read_value = true); IVariableV1 assign_sub_lazy_load(Tensor delta, string name = null); diff --git a/src/TensorFlowNET.Core/Variables/RefVariable.cs b/src/TensorFlowNET.Core/Variables/RefVariable.cs index 67c12c427..7b08f3ea4 100644 --- a/src/TensorFlowNET.Core/Variables/RefVariable.cs +++ b/src/TensorFlowNET.Core/Variables/RefVariable.cs @@ -20,11 +20,12 @@ limitations under the License. using System.Collections.Generic; using System.Linq; using static Tensorflow.Binding; +using Tensorflow.Train; namespace Tensorflow { [Obsolete] - public partial class RefVariable : IVariableV1, IProtoBuf + public partial class RefVariable: Trackable, IVariableV1, IProtoBuf { protected string _name; public string UniqueId => _name; @@ -56,6 +57,7 @@ public partial class RefVariable : IVariableV1, IProtoBuf _variable.name; public Tensor eval() => _variable; + public bool Trainable => _trainable; public RefVariable(object initial_value = null, bool trainable = true, diff --git a/src/TensorFlowNET.Core/Variables/ResourceVariable.cs b/src/TensorFlowNET.Core/Variables/ResourceVariable.cs index b31960c73..1645d7130 100644 --- a/src/TensorFlowNET.Core/Variables/ResourceVariable.cs +++ b/src/TensorFlowNET.Core/Variables/ResourceVariable.cs @@ -17,7 +17,9 @@ limitations under the License. using Google.Protobuf; using System; using System.Collections.Generic; +using Tensorflow.Checkpoint; using Tensorflow.NumPy; +using Tensorflow.Train; using static Tensorflow.Binding; namespace Tensorflow @@ -39,6 +41,7 @@ public ResourceVariable(object initial_value = null, VariableAggregation aggregation = VariableAggregation.None, Shape shape = null) { + Aggregation = aggregation; if (variable_def != null) { if (initial_value != null) diff --git a/src/TensorFlowNET.Core/Variables/UninitializedVariable.cs b/src/TensorFlowNET.Core/Variables/UninitializedVariable.cs new file mode 100644 index 000000000..6c0349950 --- /dev/null +++ b/src/TensorFlowNET.Core/Variables/UninitializedVariable.cs @@ -0,0 +1,70 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Gradients; +using static Tensorflow.Binding; + +namespace Tensorflow.Variables +{ + /// + /// A variable with no initializer. + /// + public sealed class UninitializedVariable: BaseResourceVariable + { + // TODO: complete the arg list. + public UninitializedVariable( + bool trainable = true, + string caching_device = "", + string name = null, + TF_DataType dtype = TF_DataType.DtInvalid, + VariableAggregation aggregation = VariableAggregation.None, + Shape shape = null, + Tensor extra_handle_data = null) + { + string unique_id = ""; + string handle_name = ""; + tf_with(ops.init_scope(), (x) => + { + _in_graph_mode = !tf.Context.executing_eagerly(); + tf_with(ops.name_scope(name, "Variable", skip_on_eager: false), name => + { + handle_name = ops.name_from_scope_name(name); + string? shared_name; + if (_in_graph_mode) + { + shared_name = handle_name; + unique_id = shared_name; + } + else + { + unique_id = $"{handle_name}-{ops.uid()}"; + shared_name = null; + } + var handle = resource_variable_ops.variable_handle_from_shape_and_dtype( + shape, dtype, shared_name, name, _in_graph_mode, extra_handle_data); + // skip the assignment of `handle._parent_trackable` because of lack of API. + // skip the assignment of `handle._name` and `handle._unique_id` because of accessability. + + if (_in_graph_mode) + { + tf_with(ops.name_scope("Read"), _ => + { + tf.device(handle.Device); + var value = gen_resource_variable_ops.read_variable_op(handle, dtype); + // _maybe_set_handle_data(dtype, handle, value) + _graph_element = value; + }); + ops.add_to_collection(ops.GraphKeys.GLOBAL_VARIABLES_, this); + } + else + { + _graph_element = null; + } + }); + }); + _shape = shape; + _dtype = dtype; + base.__init__(trainable, handle, unique_id: unique_id, handle_name: handle_name); + } + } +} diff --git a/src/TensorFlowNET.Core/ops.cs b/src/TensorFlowNET.Core/ops.cs index 95e8db577..bf5ae7bee 100644 --- a/src/TensorFlowNET.Core/ops.cs +++ b/src/TensorFlowNET.Core/ops.cs @@ -566,5 +566,23 @@ public static bool executing_eagerly_outside_functions() else throw new NotImplementedException(""); } + + public static bool inside_function() + { + return get_default_graph().building_function; + } + + public static void dismantle_graph(Graph graph) + { + + } + + public class NullContextManager: IDisposable + { + public void Dispose() + { + + } + } } } diff --git a/src/TensorFlowNET.Keras/Activations.cs b/src/TensorFlowNET.Keras/Activations.cs new file mode 100644 index 000000000..444c783e0 --- /dev/null +++ b/src/TensorFlowNET.Keras/Activations.cs @@ -0,0 +1,82 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Operations.Activation; +using static Tensorflow.Binding; + +namespace Tensorflow.Keras +{ + public class Activations + { + private static Dictionary _nameActivationMap; + private static Dictionary _activationNameMap; + + private static Activation _linear = (features, name) => features; + private static Activation _relu = (features, name) + => tf.Context.ExecuteOp("Relu", name, new ExecuteOpArgs(features)); + private static Activation _sigmoid = (features, name) + => tf.Context.ExecuteOp("Sigmoid", name, new ExecuteOpArgs(features)); + private static Activation _softmax = (features, name) + => tf.Context.ExecuteOp("Softmax", name, new ExecuteOpArgs(features)); + private static Activation _tanh = (features, name) + => tf.Context.ExecuteOp("Tanh", name, new ExecuteOpArgs(features)); + + /// + /// Register the name-activation mapping in this static class. + /// + /// + /// + private static void RegisterActivation(string name, Activation activation) + { + _nameActivationMap[name] = activation; + _activationNameMap[activation] = name; + } + + static Activations() + { + _nameActivationMap = new Dictionary(); + _activationNameMap= new Dictionary(); + + RegisterActivation("relu", _relu); + RegisterActivation("linear", _linear); + RegisterActivation("sigmoid", _sigmoid); + RegisterActivation("softmax", _softmax); + RegisterActivation("tanh", _tanh); + } + + public Activation Linear => _linear; + + public Activation Relu => _relu; + + public Activation Sigmoid => _sigmoid; + + public Activation Softmax => _softmax; + + public Activation Tanh => _tanh; + + + public static Activation GetActivationByName(string name) + { + if (!_nameActivationMap.TryGetValue(name, out var res)) + { + throw new Exception($"Activation {name} not found"); + } + else + { + return res; + } + } + + public static string GetNameByActivation(Activation activation) + { + if(!_activationNameMap.TryGetValue(activation, out var name)) + { + throw new Exception($"Activation {activation} not found"); + } + else + { + return name; + } + } + } +} diff --git a/src/TensorFlowNET.Keras/Activations/Activations.Linear.cs b/src/TensorFlowNET.Keras/Activations/Activations.Linear.cs deleted file mode 100644 index acd4de6e7..000000000 --- a/src/TensorFlowNET.Keras/Activations/Activations.Linear.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace Tensorflow.Keras -{ - public partial class Activations - { - /// - /// Linear activation function (pass-through). - /// - public Activation Linear = (features, name) => features; - } -} diff --git a/src/TensorFlowNET.Keras/Activations/Activations.Relu.cs b/src/TensorFlowNET.Keras/Activations/Activations.Relu.cs deleted file mode 100644 index dfebfb297..000000000 --- a/src/TensorFlowNET.Keras/Activations/Activations.Relu.cs +++ /dev/null @@ -1,10 +0,0 @@ -using static Tensorflow.Binding; - -namespace Tensorflow.Keras -{ - public partial class Activations - { - public Activation Relu = (features, name) - => tf.Context.ExecuteOp("Relu", name, new ExecuteOpArgs(features)); - } -} diff --git a/src/TensorFlowNET.Keras/Activations/Activations.Sigmoid.cs b/src/TensorFlowNET.Keras/Activations/Activations.Sigmoid.cs deleted file mode 100644 index ad900bdef..000000000 --- a/src/TensorFlowNET.Keras/Activations/Activations.Sigmoid.cs +++ /dev/null @@ -1,11 +0,0 @@ -using System; -using static Tensorflow.Binding; - -namespace Tensorflow.Keras -{ - public partial class Activations - { - public Activation Sigmoid = (features, name) - => tf.Context.ExecuteOp("Sigmoid", name, new ExecuteOpArgs(features)); - } -} diff --git a/src/TensorFlowNET.Keras/Activations/Activations.Softmax.cs b/src/TensorFlowNET.Keras/Activations/Activations.Softmax.cs deleted file mode 100644 index 02d86acea..000000000 --- a/src/TensorFlowNET.Keras/Activations/Activations.Softmax.cs +++ /dev/null @@ -1,11 +0,0 @@ -using System; -using static Tensorflow.Binding; - -namespace Tensorflow.Keras -{ - public partial class Activations - { - public Activation Softmax = (features, name) - => tf.Context.ExecuteOp("Softmax", name, new ExecuteOpArgs(features)); - } -} diff --git a/src/TensorFlowNET.Keras/Activations/Activations.Tanh.cs b/src/TensorFlowNET.Keras/Activations/Activations.Tanh.cs deleted file mode 100644 index 33dc5ba62..000000000 --- a/src/TensorFlowNET.Keras/Activations/Activations.Tanh.cs +++ /dev/null @@ -1,11 +0,0 @@ -using System; -using static Tensorflow.Binding; - -namespace Tensorflow.Keras -{ - public partial class Activations - { - public Activation Tanh = (features, name) - => tf.Context.ExecuteOp("Tanh", name, new ExecuteOpArgs(features)); - } -} diff --git a/src/TensorFlowNET.Keras/Engine/Functional.GetConfig.cs b/src/TensorFlowNET.Keras/Engine/Functional.GetConfig.cs index 23c40fbff..3aeb3200d 100644 --- a/src/TensorFlowNET.Keras/Engine/Functional.GetConfig.cs +++ b/src/TensorFlowNET.Keras/Engine/Functional.GetConfig.cs @@ -11,7 +11,7 @@ namespace Tensorflow.Keras.Engine { public partial class Functional { - public ModelConfig get_config() + public override IKerasConfig get_config() { return get_network_config(); } @@ -25,7 +25,7 @@ ModelConfig get_network_config() { Name = name }; - + var node_conversion_map = new Dictionary(); foreach (var layer in _self_tracked_trackables) { @@ -42,23 +42,26 @@ ModelConfig get_network_config() } var layer_configs = new List(); - foreach (var layer in _self_tracked_trackables) + using (SharedObjectSavingScope.Enter()) { - var filtered_inbound_nodes = new List(); - foreach (var (original_node_index, node) in enumerate(layer.InboundNodes)) + foreach (var layer in _self_tracked_trackables) { - var node_key = _make_node_key(layer.Name, original_node_index); - if (NetworkNodes.Contains(node_key) && !node.is_input) + var filtered_inbound_nodes = new List(); + foreach (var (original_node_index, node) in enumerate(layer.InboundNodes)) { - var node_data = node.serialize(_make_node_key, node_conversion_map); - filtered_inbound_nodes.append(node_data); + var node_key = _make_node_key(layer.Name, original_node_index); + if (NetworkNodes.Contains(node_key) && !node.is_input) + { + var node_data = node.serialize(_make_node_key, node_conversion_map); + filtered_inbound_nodes.append(node_data); + } } - } - var layer_config = generic_utils.serialize_keras_object(layer); - layer_config.Name = layer.Name; - layer_config.InboundNodes = filtered_inbound_nodes; - layer_configs.Add(layer_config); + var layer_config = generic_utils.serialize_layer_to_config(layer); + layer_config.Name = layer.Name; + layer_config.InboundNodes = filtered_inbound_nodes; + layer_configs.Add(layer_config); + } } config.Layers = layer_configs; diff --git a/src/TensorFlowNET.Keras/Engine/Functional.cs b/src/TensorFlowNET.Keras/Engine/Functional.cs index 09a31b948..44eaef534 100644 --- a/src/TensorFlowNET.Keras/Engine/Functional.cs +++ b/src/TensorFlowNET.Keras/Engine/Functional.cs @@ -2,7 +2,9 @@ using System.Collections.Generic; using System.Linq; using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.Saving.SavedModel; using Tensorflow.Keras.Utils; +using Tensorflow.Train; using static Tensorflow.Binding; namespace Tensorflow.Keras.Engine @@ -20,6 +22,30 @@ public partial class Functional : Model Dictionary tensor_usage_count; + /// + /// Dictionary of layer dependencies to be included in the checkpoint. + /// + public IDictionary LayerCheckpointDependencies + { + get + { + int weight_layer_index = 0; + Dictionary dependencies = new(); + for(int i = 0; i < Layers.Count; i++) + { + var layer = Layers[i]; + var weights = layer.TrainableWeights.concat(layer.NonTrainableWeights).ToList(); + if(weights.Count > 0) + { + dependencies[$"layer_with_weights-{weight_layer_index}"] = layer; + weight_layer_index++; + } + dependencies[$"layer-{i}"] = layer; + } + return dependencies; + } + } + public Functional(Tensors inputs, Tensors outputs, string name = null) : base(new ModelArgs { @@ -44,6 +70,7 @@ protected void _init_graph_network(Tensors inputs, Tensors outputs) this.inputs = inputs; this.outputs = outputs; built = true; + _buildInputShape = inputs.shape; if (outputs.Any(x => x.KerasHistory == null)) base_layer_utils.create_keras_history(outputs); @@ -325,5 +352,28 @@ protected override Tensors Call(Tensors inputs, Tensor state = null, bool? train return output_tensors; } + + public override IDictionary _trackable_children(SaveType save_type = SaveType.CHECKPOINT, IDictionary>? cache = null) + { + return LayerCheckpointDependencies.ToDictionary(x => x.Key, x => x.Value.GetTrackable()).Concat(base._trackable_children(save_type, cache)) + .ToDictionary(x => x.Key, x => x.Value); + } + + protected override void _init_set_name(string name, bool zero_based = true) + { + if (string.IsNullOrEmpty(name)) + { + string class_name = GetType().Name; + if (this.GetType() == typeof(Functional)) + { + class_name = "Model"; + } + this.name = base_layer_utils.unique_layer_name(generic_utils.to_snake_case(class_name), zero_based: zero_based); + } + else + { + this.name = name; + } + } } } diff --git a/src/TensorFlowNET.Keras/Engine/Layer.Serialize.cs b/src/TensorFlowNET.Keras/Engine/Layer.Serialize.cs new file mode 100644 index 000000000..fc405d872 --- /dev/null +++ b/src/TensorFlowNET.Keras/Engine/Layer.Serialize.cs @@ -0,0 +1,32 @@ +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Tensorflow.Keras.Saving.SavedModel; +using Tensorflow.Train; + +namespace Tensorflow.Keras.Engine; + +public abstract partial class Layer +{ + public virtual SavedModelSaver TrackableSavedModelSaver => new LayerSavedModelSaver(this); + + public override string ObjectIdentifier => TrackableSavedModelSaver.ObjectIdentifier; + + public string TrackingMetadata => TrackableSavedModelSaver.TrackingMetadata; + + public override IDictionary _trackable_children(SaveType save_type = SaveType.CHECKPOINT, IDictionary>? cache = null) + { + IDictionary children; + if (save_type == SaveType.SAVEDMODEL) + { + Debug.Assert(cache is not null); + children = TrackableSavedModelSaver.trackable_children(cache); + } + else + { + children = new Dictionary(); + } + + return children.Concat(base._trackable_children(save_type, cache)).ToDictionary(x => x.Key, x => x.Value); + } +} \ No newline at end of file diff --git a/src/TensorFlowNET.Keras/Engine/Layer.cs b/src/TensorFlowNET.Keras/Engine/Layer.cs index ba40b1a22..31b37d681 100644 --- a/src/TensorFlowNET.Keras/Engine/Layer.cs +++ b/src/TensorFlowNET.Keras/Engine/Layer.cs @@ -49,6 +49,8 @@ public abstract partial class Layer : AutoTrackable, ILayer public bool Built => built; public bool Trainable => args.Trainable; public TF_DataType DType => args.DType; + public bool AutoCast => args.Autocast; + public IRegularizer ActivityRegularizer => args.ActivityRegularizer; /// /// A stateful layer is a layer whose updates are run during inference too, @@ -59,6 +61,7 @@ public abstract partial class Layer : AutoTrackable, ILayer /// Provides information about which inputs are compatible with the layer. /// protected InputSpec inputSpec; + public InputSpec InputSpec => inputSpec; bool dynamic = true; public bool SupportsMasking { get; set; } protected List _trainable_weights; @@ -77,6 +80,8 @@ public abstract partial class Layer : AutoTrackable, ILayer protected bool computePreviousMask; protected List updates; public Shape BatchInputShape => args.BatchInputShape; + protected TensorShapeConfig _buildInputShape = null; + public TensorShapeConfig BuildInputShape => _buildInputShape; List inboundNodes; public List InboundNodes => inboundNodes; @@ -86,9 +91,29 @@ public abstract partial class Layer : AutoTrackable, ILayer ThreadLocal callContext = new ThreadLocal(); public CallContext CallContext => callContext.Value; - public Tensor[] input => inboundNodes[0].input_tensors; + public Tensor[] input + { + get + { + if(inboundNodes is not null && inboundNodes.Count > 0) + { + return inboundNodes[0].input_tensors; + } + return null; + } + } public Dictionary> NodesByDepth { get; set; } - public Shape OutputShape => inboundNodes[0].Outputs.shape; + public Shape OutputShape + { + get + { + if(inboundNodes is not null && inboundNodes.Count > 0) + { + return inboundNodes[0].Outputs.shape; + } + return null; + } + } protected List _self_tracked_trackables; public Layer(LayerArgs args) @@ -162,7 +187,7 @@ private Tensor compute_mask(Tensor inputs, Tensor mask = null) /// /// /// - /// + /// /// protected virtual Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) { @@ -201,6 +226,7 @@ protected void MaybeBuild(Tensors inputs) public virtual void build(Shape input_shape) { + _buildInputShape = input_shape; built = true; } @@ -286,7 +312,9 @@ public List weights } } - public virtual LayerArgs get_config() + public List Variables => weights; + + public virtual IKerasConfig get_config() => args; } } diff --git a/src/TensorFlowNET.Keras/Engine/Model.Fit.cs b/src/TensorFlowNET.Keras/Engine/Model.Fit.cs index bc2c2cea6..966853809 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Fit.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Fit.cs @@ -33,6 +33,11 @@ public History fit(NDArray x, NDArray y, int workers = 1, bool use_multiprocessing = false) { + if (x.dims[0] != y.dims[0]) + { + throw new InvalidArgumentError( + $"The array x and y should have same value at dim 0, but got {x.dims[0]} and {y.dims[0]}"); + } int train_count = Convert.ToInt32(x.dims[0] * (1 - validation_split)); var train_x = x[new Slice(0, train_count)]; var train_y = y[new Slice(0, train_count)]; diff --git a/src/TensorFlowNET.Keras/Engine/Model.Save.cs b/src/TensorFlowNET.Keras/Engine/Model.Save.cs index c287309d4..a1e891f98 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Save.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Save.cs @@ -1,5 +1,8 @@ using System.Collections.Generic; +using Tensorflow.Functions; using Tensorflow.Keras.Metrics; +using Tensorflow.Keras.Saving; +using Tensorflow.Keras.Saving.SavedModel; using Tensorflow.ModelSaving; namespace Tensorflow.Keras.Engine @@ -18,9 +21,21 @@ public void save(string filepath, bool overwrite = true, bool include_optimizer = true, string save_format = "tf", - SaveOptions options = null) + SaveOptions? options = null, + ConcreteFunction? signatures = null, + bool save_traces = true) { - saver.save(this, filepath); + if (save_format != "tf") + { + saver.save(this, filepath); + } + else + { + using (SharedObjectSavingScope.Enter()) + { + KerasSavedModelUtils.Save(this, filepath, overwrite, include_optimizer, signatures, options, save_traces); + } + } } } } diff --git a/src/TensorFlowNET.Keras/Engine/Model.cs b/src/TensorFlowNET.Keras/Engine/Model.cs index 9bab9bd2f..dfe5b05f3 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.cs @@ -4,6 +4,8 @@ using Tensorflow.Keras.Engine.DataAdapters; using Tensorflow.Keras.Losses; using Tensorflow.Keras.Optimizers; +using Tensorflow.Keras.Saving.SavedModel; +using Tensorflow.Train; using static Tensorflow.Binding; using static Tensorflow.KerasApi; @@ -34,6 +36,13 @@ public partial class Model : Layer, IModel IVariableV1 _predict_counter; bool _base_model_initialized; bool stop_training; + DataHandler data_handler; + + public OptimizerV2 Optimizer + { + get => optimizer; + set => optimizer = value; + } public Model(ModelArgs args) : base(args) @@ -101,5 +110,15 @@ public override List TrainableVariables return variables; } } + + public override IDictionary _trackable_children(SaveType save_type = SaveType.CHECKPOINT, IDictionary>? cache = null) + { + if(save_type == SaveType.SAVEDMODEL) + { + //TODO: deal with `train_function`, `test_function`, `predict_function`, `train_tf_function`. + } + var children = base._trackable_children(save_type, cache); + return children; + } } } diff --git a/src/TensorFlowNET.Keras/Layers/Activation/ELU.cs b/src/TensorFlowNET.Keras/Layers/Activation/ELU.cs index 6e790a26f..45f64720f 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/ELU.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/ELU.cs @@ -25,6 +25,7 @@ public override void build(Shape input_shape) { throw new ValueError("Alpha must be a number greater than 0."); } + _buildInputShape = input_shape; built = true; } diff --git a/src/TensorFlowNET.Keras/Layers/Activation/Exponential.cs b/src/TensorFlowNET.Keras/Layers/Activation/Exponential.cs index aba175de9..2fd2caee1 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/Exponential.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/Exponential.cs @@ -14,6 +14,7 @@ public Exponential(LayerArgs args) : base(args) } public override void build(Shape input_shape) { + _buildInputShape = input_shape; built = true; } protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) diff --git a/src/TensorFlowNET.Keras/Layers/Activation/SELU.cs b/src/TensorFlowNET.Keras/Layers/Activation/SELU.cs index b12d7deec..1ef8d0e58 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/SELU.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/SELU.cs @@ -16,10 +16,11 @@ public SELU ( LayerArgs args ) : base(args) { // SELU has no arguments } public override void build(Shape input_shape) { - if ( alpha < 0f ) { - throw new ValueError("Alpha must be a number greater than 0."); - } - built = true; + if ( alpha < 0f ) { + throw new ValueError("Alpha must be a number greater than 0."); + } + _buildInputShape = input_shape; + built = true; } protected override Tensors Call ( Tensors inputs, Tensor state = null, bool? training = null ) { Tensor output = inputs; diff --git a/src/TensorFlowNET.Keras/Layers/Attention/Attention.cs b/src/TensorFlowNET.Keras/Layers/Attention/Attention.cs index 6f6dd7e85..c51316308 100644 --- a/src/TensorFlowNET.Keras/Layers/Attention/Attention.cs +++ b/src/TensorFlowNET.Keras/Layers/Attention/Attention.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Linq; using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.Saving; namespace Tensorflow.Keras.Layers { @@ -146,7 +147,7 @@ public override Tensor _calculate_scores(Tensor query, Tensor key) return scores; } - public override LayerArgs get_config() => this.args; + public override IKerasConfig get_config() => this.args; //var config = new Dictionary { // { // "use_scale", diff --git a/src/TensorFlowNET.Keras/Layers/Attention/BaseDenseAttention.cs b/src/TensorFlowNET.Keras/Layers/Attention/BaseDenseAttention.cs index 3f618b5db..1348e19cf 100644 --- a/src/TensorFlowNET.Keras/Layers/Attention/BaseDenseAttention.cs +++ b/src/TensorFlowNET.Keras/Layers/Attention/BaseDenseAttention.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.Linq; +using Tensorflow.Keras.Saving; /// /// Base class for attention layers that can be used in sequence DNN/CNN models. @@ -252,6 +253,6 @@ public static Tensor _merge_masks(Tensor x, Tensor y) return tf.logical_and(x, y); } - public override LayerArgs get_config() => this.args; + public override IKerasConfig get_config() => this.args; } } diff --git a/src/TensorFlowNET.Keras/Layers/Attention/MultiHeadAttention.cs b/src/TensorFlowNET.Keras/Layers/Attention/MultiHeadAttention.cs index 1b82e0a96..701724d5b 100644 --- a/src/TensorFlowNET.Keras/Layers/Attention/MultiHeadAttention.cs +++ b/src/TensorFlowNET.Keras/Layers/Attention/MultiHeadAttention.cs @@ -1,4 +1,5 @@ using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.ArgsDefinition.Core; using Tensorflow.Keras.Engine; using Tensorflow.NumPy; using static Tensorflow.Binding; diff --git a/src/TensorFlowNET.Keras/Layers/Convolution/Conv2DTranspose.cs b/src/TensorFlowNET.Keras/Layers/Convolution/Conv2DTranspose.cs index e0a337caa..b8286be67 100644 --- a/src/TensorFlowNET.Keras/Layers/Convolution/Conv2DTranspose.cs +++ b/src/TensorFlowNET.Keras/Layers/Convolution/Conv2DTranspose.cs @@ -49,6 +49,7 @@ public override void build(Shape input_shape) initializer: bias_initializer, trainable: true); built = true; + _buildInputShape = input_shape; } protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) diff --git a/src/TensorFlowNET.Keras/Layers/Convolution/Convolutional.cs b/src/TensorFlowNET.Keras/Layers/Convolution/Convolutional.cs index 912a429b7..933aa9cf1 100644 --- a/src/TensorFlowNET.Keras/Layers/Convolution/Convolutional.cs +++ b/src/TensorFlowNET.Keras/Layers/Convolution/Convolutional.cs @@ -98,6 +98,7 @@ public override void build(Shape input_shape) name: tf_op_name); built = true; + _buildInputShape = input_shape; } protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = false) diff --git a/src/TensorFlowNET.Keras/Layers/Core/Dense.cs b/src/TensorFlowNET.Keras/Layers/Core/Dense.cs index e4c227456..ca8007d09 100644 --- a/src/TensorFlowNET.Keras/Layers/Core/Dense.cs +++ b/src/TensorFlowNET.Keras/Layers/Core/Dense.cs @@ -43,6 +43,7 @@ public Dense(DenseArgs args) : public override void build(Shape input_shape) { + _buildInputShape = input_shape; var last_dim = input_shape.dims.Last(); var axes = new Dictionary(); axes[-1] = (int)last_dim; diff --git a/src/TensorFlowNET.Keras/Layers/Core/EinsumDense.cs b/src/TensorFlowNET.Keras/Layers/Core/EinsumDense.cs index 0f387570b..af71ddf9f 100644 --- a/src/TensorFlowNET.Keras/Layers/Core/EinsumDense.cs +++ b/src/TensorFlowNET.Keras/Layers/Core/EinsumDense.cs @@ -4,8 +4,8 @@ using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; -using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; +using Tensorflow.Keras.ArgsDefinition.Core; namespace Tensorflow.Keras.Layers { diff --git a/src/TensorFlowNET.Keras/Layers/Core/Embedding.cs b/src/TensorFlowNET.Keras/Layers/Core/Embedding.cs index 79f4e5ce9..606f387bb 100644 --- a/src/TensorFlowNET.Keras/Layers/Core/Embedding.cs +++ b/src/TensorFlowNET.Keras/Layers/Core/Embedding.cs @@ -62,6 +62,7 @@ public override void build(Shape input_shape) name: "embeddings"); tf.Context.graph_mode(); built = true; + _buildInputShape = input_shape; } protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) diff --git a/src/TensorFlowNET.Keras/Layers/Core/InputLayer.cs b/src/TensorFlowNET.Keras/Layers/Core/InputLayer.cs index 6b064716f..03b4b742a 100644 --- a/src/TensorFlowNET.Keras/Layers/Core/InputLayer.cs +++ b/src/TensorFlowNET.Keras/Layers/Core/InputLayer.cs @@ -18,6 +18,7 @@ limitations under the License. using Tensorflow.Framework.Models; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; +using Tensorflow.Keras.Saving.SavedModel; using static Tensorflow.Binding; using static Tensorflow.KerasApi; @@ -105,5 +106,7 @@ public static InputLayer from_config(LayerArgs args) { return new InputLayer(args as InputLayerArgs); } + + public override SavedModelSaver TrackableSavedModelSaver => new InputLayerSavedModelSaver(this); } } diff --git a/src/TensorFlowNET.Keras/Layers/Cropping/Cropping2D.cs b/src/TensorFlowNET.Keras/Layers/Cropping/Cropping2D.cs deleted file mode 100644 index 6cb03e1e0..000000000 --- a/src/TensorFlowNET.Keras/Layers/Cropping/Cropping2D.cs +++ /dev/null @@ -1,113 +0,0 @@ -using Tensorflow.Keras.ArgsDefinition; -using Tensorflow.Keras.Engine; - -namespace Tensorflow.Keras.Layers { - /// - /// Crop the input along axis 1 and 2. - /// For example: - /// shape (1, 5, 5, 5) -- crop2D ((1, 2), (1, 3)) --> shape (1, 2, 1, 5) - /// - public class Cropping2D : Layer { - Cropping2DArgs args; - public Cropping2D ( Cropping2DArgs args ) : base(args) { - this.args = args; - } - public override void build(Shape input_shape) { - built = true; - } - protected override Tensors Call ( Tensors inputs, Tensor state = null, bool? training = null ) { - Tensor output = inputs; - if ( output.rank != 4 ) { - // throw an ValueError exception - throw new ValueError("Expected dim=4, found dim=" + output.rank); - } - if ( args.cropping.shape == new Shape(1) ) { - int crop = args.cropping[0]; - if ( args.data_format == Cropping2DArgs.DataFormat.channels_last ) { - output = output[new Slice(), - new Slice(crop, ( int ) output.shape[1] - crop), - new Slice(crop, ( int ) output.shape[2] - crop), - new Slice()]; - } - else { - output = output[new Slice(), - new Slice(), - new Slice(crop, ( int ) output.shape[2] - crop), - new Slice(crop, ( int ) output.shape[3] - crop)]; - } - } - // a tuple of 2 integers - else if ( args.cropping.shape == new Shape(2) ) { - int crop_1 = args.cropping[0]; - int crop_2 = args.cropping[1]; - if ( args.data_format == Cropping2DArgs.DataFormat.channels_last ) { - output = output[new Slice(), - new Slice(crop_1, ( int ) output.shape[1] - crop_1), - new Slice(crop_2, ( int ) output.shape[2] - crop_2), - new Slice()]; - } - else { - output = output[new Slice(), - new Slice(), - new Slice(crop_1, ( int ) output.shape[2] - crop_1), - new Slice(crop_2, ( int ) output.shape[3] - crop_2)]; - } - } - else if ( args.cropping.shape[0] == 2 && args.cropping.shape[1] == 2 ) { - int x_start = args.cropping[0, 0], x_end = args.cropping[0, 1]; - int y_start = args.cropping[1, 0], y_end = args.cropping[1, 1]; - if ( args.data_format == Cropping2DArgs.DataFormat.channels_last ) { - output = output[new Slice(), - new Slice(x_start, ( int ) output.shape[1] - x_end), - new Slice(y_start, ( int ) output.shape[2] - y_end), - new Slice()]; - } - else { - output = output[new Slice(), - new Slice(), - new Slice(x_start, ( int ) output.shape[2] - x_end), - new Slice(y_start, ( int ) output.shape[3] - y_end) - ]; - } - } - return output; - } - - public override Shape ComputeOutputShape ( Shape input_shape ) { - if ( args.cropping.shape == new Shape(1) ) { - int crop = args.cropping[0]; - if ( args.data_format == Cropping2DArgs.DataFormat.channels_last ) { - return new Shape(( int ) input_shape[0], ( int ) input_shape[1] - crop * 2, ( int ) input_shape[2] - crop * 2, ( int ) input_shape[3]); - } - else { - return new Shape(( int ) input_shape[0], ( int ) input_shape[1], ( int ) input_shape[2] - crop * 2, ( int ) input_shape[3] - crop * 2); - } - } - // a tuple of 2 integers - else if ( args.cropping.shape == new Shape(2) ) { - int crop_1 = args.cropping[0], crop_2 = args.cropping[1]; - if ( args.data_format == Cropping2DArgs.DataFormat.channels_last ) { - return new Shape(( int ) input_shape[0], ( int ) input_shape[1] - crop_1 * 2, ( int ) input_shape[2] - crop_2 * 2, ( int ) input_shape[3]); - } - else { - return new Shape(( int ) input_shape[0], ( int ) input_shape[1], ( int ) input_shape[2] - crop_1 * 2, ( int ) input_shape[3] - crop_2 * 2); - } - } - else if ( args.cropping.shape == new Shape(2, 2) ) { - int crop_1_start = args.cropping[0, 0], crop_1_end = args.cropping[0, 1]; - int crop_2_start = args.cropping[1, 0], crop_2_end = args.cropping[1, 1]; - if ( args.data_format == Cropping2DArgs.DataFormat.channels_last ) { - return new Shape(( int ) input_shape[0], ( int ) input_shape[1] - crop_1_start - crop_1_end, - ( int ) input_shape[2] - crop_2_start - crop_2_end, ( int ) input_shape[3]); - } - else { - return new Shape(( int ) input_shape[0], ( int ) input_shape[1], - ( int ) input_shape[2] - crop_1_start - crop_1_end, ( int ) input_shape[3] - crop_2_start - crop_2_end); - } - } - else { - throw new ValueError(); - } - } - } -} diff --git a/src/TensorFlowNET.Keras/Layers/Cropping/Cropping3D.cs b/src/TensorFlowNET.Keras/Layers/Cropping/Cropping3D.cs deleted file mode 100644 index 2d6751bf9..000000000 --- a/src/TensorFlowNET.Keras/Layers/Cropping/Cropping3D.cs +++ /dev/null @@ -1,123 +0,0 @@ -using Tensorflow.Keras.ArgsDefinition; -using Tensorflow.Keras.Engine; - -namespace Tensorflow.Keras.Layers { - /// - /// Similar to copping 2D - /// - public class Cropping3D : Layer { - Cropping3DArgs args; - public Cropping3D ( Cropping3DArgs args ) : base(args) { - this.args = args; - } - - public override void build(Shape input_shape) { - built = true; - } - - protected override Tensors Call ( Tensors inputs, Tensor state = null, bool? training = null ) { - Tensor output = inputs; - if ( output.rank != 5 ) { - // throw an ValueError exception - throw new ValueError("Expected dim=5, found dim=" + output.rank); - } - - if ( args.cropping.shape == new Shape(1) ) { - int crop = args.cropping[0]; - if ( args.data_format == Cropping3DArgs.DataFormat.channels_last ) { - output = output[new Slice(), - new Slice(crop, ( int ) output.shape[1] - crop), - new Slice(crop, ( int ) output.shape[2] - crop), - new Slice(crop, ( int ) output.shape[3] - crop), - new Slice()]; - } - else { - output = output[new Slice(), - new Slice(), - new Slice(crop, ( int ) output.shape[2] - crop), - new Slice(crop, ( int ) output.shape[3] - crop), - new Slice(crop, ( int ) output.shape[4] - crop)]; - } - - } - // int[1][3] equivalent to a tuple of 3 integers - else if ( args.cropping.shape == new Shape(3) ) { - var crop_1 = args.cropping[0]; - var crop_2 = args.cropping[1]; - var crop_3 = args.cropping[2]; - if ( args.data_format == Cropping3DArgs.DataFormat.channels_last ) { - output = output[new Slice(), - new Slice(crop_1, ( int ) output.shape[1] - crop_1), - new Slice(crop_2, ( int ) output.shape[2] - crop_2), - new Slice(crop_3, ( int ) output.shape[3] - crop_3), - new Slice()]; - } - else { - output = output[new Slice(), - new Slice(), - new Slice(crop_1, ( int ) output.shape[2] - crop_1), - new Slice(crop_2, ( int ) output.shape[3] - crop_2), - new Slice(crop_3, ( int ) output.shape[4] - crop_3)]; - } - } - else if ( args.cropping.shape[0] == 3 && args.cropping.shape[1] == 2 ) { - int x = args.cropping[0, 0], x_end = args.cropping[0, 1]; - int y = args.cropping[1, 0], y_end = args.cropping[1, 1]; - int z = args.cropping[2, 0], z_end = args.cropping[2, 1]; - if ( args.data_format == Cropping3DArgs.DataFormat.channels_last ) { - output = output[new Slice(), - new Slice(x, ( int ) output.shape[1] - x_end), - new Slice(y, ( int ) output.shape[2] - y_end), - new Slice(z, ( int ) output.shape[3] - z_end), - new Slice()]; - } - else { - output = output[new Slice(), - new Slice(), - new Slice(x, ( int ) output.shape[2] - x_end), - new Slice(y, ( int ) output.shape[3] - y_end), - new Slice(z, ( int ) output.shape[4] - z_end) - ]; - } - } - return output; - } - public override Shape ComputeOutputShape ( Shape input_shape ) { - if ( args.cropping.shape == new Shape(1) ) { - int crop = args.cropping[0]; - if ( args.data_format == Cropping3DArgs.DataFormat.channels_last ) { - return new Shape(( int ) input_shape[0], ( int ) input_shape[1] - crop * 2, ( int ) input_shape[2] - crop * 2, ( int ) input_shape[3] - crop * 2, ( int ) input_shape[4]); - } - else { - return new Shape(( int ) input_shape[0], ( int ) input_shape[1], ( int ) input_shape[2] - crop * 2, ( int ) input_shape[3] - crop * 2, ( int ) input_shape[4] - crop * 2); - } - } - // int[1][3] equivalent to a tuple of 3 integers - else if ( args.cropping.shape == new Shape(3) ) { - var crop_start_1 = args.cropping[0]; - var crop_start_2 = args.cropping[1]; - var crop_start_3 = args.cropping[2]; - if ( args.data_format == Cropping3DArgs.DataFormat.channels_last ) { - return new Shape(( int ) input_shape[0], ( int ) input_shape[1] - crop_start_1 * 2, ( int ) input_shape[2] - crop_start_2 * 2, ( int ) input_shape[3] - crop_start_3 * 2, ( int ) input_shape[4]); - } - else { - return new Shape(( int ) input_shape[0], ( int ) input_shape[1], ( int ) input_shape[2] - crop_start_1 * 2, ( int ) input_shape[3] - crop_start_2 * 2, ( int ) input_shape[4] - crop_start_3 * 2); - } - } - else if ( args.cropping.shape == new Shape(3, 2) ) { - int x = args.cropping[0, 0], x_end = args.cropping[0, 1]; - int y = args.cropping[1, 0], y_end = args.cropping[1, 1]; - int z = args.cropping[2, 0], z_end = args.cropping[2, 1]; - if ( args.data_format == Cropping3DArgs.DataFormat.channels_last ) { - return new Shape(( int ) input_shape[0], ( int ) input_shape[1] - x - x_end, ( int ) input_shape[2] - y - y_end, ( int ) input_shape[3] - z - z_end, ( int ) input_shape[4]); - } - else { - return new Shape(( int ) input_shape[0], ( int ) input_shape[1], ( int ) input_shape[2] - x - x_end, ( int ) input_shape[3] - y - y_end, ( int ) input_shape[4] - z - z_end); - } - } - else { - throw new ValueError(); - } - } - } -} diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.Cropping.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.Cropping.cs index 339ddb85b..3e3442f25 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.Cropping.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.Cropping.cs @@ -2,16 +2,18 @@ using System; using System.Collections.Generic; using System.Text; -using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.Layers.Reshaping; +using Tensorflow.Keras.ArgsDefinition.Reshaping; -namespace Tensorflow.Keras.Layers { - public partial class LayersApi { +namespace Tensorflow.Keras.Layers +{ + public partial class LayersApi { /// /// Cropping layer for 1D input /// /// cropping size public ILayer Cropping1D ( NDArray cropping ) - => new Cropping1D(new CroppingArgs { + => new Cropping1D(new Cropping1DArgs { cropping = cropping }); diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index 5c1c8995d..76634918d 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -1,9 +1,8 @@ using System; using Tensorflow.Keras.ArgsDefinition; -using Tensorflow.Keras.ArgsDefinition.Lstm; +using Tensorflow.Keras.ArgsDefinition.Core; using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; -using Tensorflow.Keras.Layers.Lstm; using Tensorflow.Keras.Layers.Rnn; using static Tensorflow.Binding; using static Tensorflow.KerasApi; @@ -108,7 +107,7 @@ public ILayer Conv1D(int filters, DilationRate = dilation_rate, Groups = groups, UseBias = use_bias, - Activation = GetActivationByName(activation), + Activation = Activations.GetActivationByName(activation), KernelInitializer = GetInitializerByName(kernel_initializer), BiasInitializer = GetInitializerByName(bias_initializer) }); @@ -163,7 +162,7 @@ public ILayer Conv2D(int filters, BiasInitializer = bias_initializer == null ? tf.zeros_initializer : bias_initializer, BiasRegularizer = bias_regularizer, ActivityRegularizer = activity_regularizer, - Activation = activation ?? keras.activations.Linear + Activation = activation ?? keras.activations.Linear, }); /// @@ -210,7 +209,8 @@ public ILayer Conv2D(int filters, UseBias = use_bias, KernelInitializer = GetInitializerByName(kernel_initializer), BiasInitializer = GetInitializerByName(bias_initializer), - Activation = GetActivationByName(activation) + Activation = Activations.GetActivationByName(activation), + ActivationName = activation }); /// @@ -255,7 +255,7 @@ public ILayer Conv2DTranspose(int filters, UseBias = use_bias, KernelInitializer = GetInitializerByName(kernel_initializer), BiasInitializer = GetInitializerByName(bias_initializer), - Activation = GetActivationByName(activation) + Activation = Activations.GetActivationByName(activation) }); /// @@ -300,7 +300,8 @@ public ILayer Dense(int units) => new Dense(new DenseArgs { Units = units, - Activation = GetActivationByName("linear") + Activation = Activations.GetActivationByName("linear"), + ActivationName = "linear" }); /// @@ -320,7 +321,8 @@ public ILayer Dense(int units, => new Dense(new DenseArgs { Units = units, - Activation = GetActivationByName(activation), + Activation = Activations.GetActivationByName(activation), + ActivationName = activation, InputShape = input_shape }); @@ -664,7 +666,7 @@ public ILayer SimpleRNN(int units, => new SimpleRNN(new SimpleRNNArgs { Units = units, - Activation = GetActivationByName(activation), + Activation = Activations.GetActivationByName(activation), KernelInitializer = GetInitializerByName(kernel_initializer), RecurrentInitializer = GetInitializerByName(recurrent_initializer), BiasInitializer = GetInitializerByName(bias_initializer), @@ -812,24 +814,7 @@ public ILayer GlobalMaxPooling1D(string data_format = "channels_last") public ILayer GlobalMaxPooling2D(string data_format = "channels_last") => new GlobalMaxPooling2D(new Pooling2DArgs { DataFormat = data_format }); - - /// - /// Get an activation function layer from its name. - /// - /// The name of the activation function. One of linear, relu, sigmoid, and tanh. - /// - - Activation GetActivationByName(string name) - => name switch - { - "linear" => keras.activations.Linear, - "relu" => keras.activations.Relu, - "sigmoid" => keras.activations.Sigmoid, - "tanh" => keras.activations.Tanh, - "softmax" => keras.activations.Softmax, - _ => throw new Exception($"Activation {name} not found") - }; - + Activation GetActivationByName(string name) => Activations.GetActivationByName(name); /// /// Get an weights initializer from its name. /// diff --git a/src/TensorFlowNET.Keras/Layers/Merging/Concatenate.cs b/src/TensorFlowNET.Keras/Layers/Merging/Concatenate.cs index 5f8217604..da7e857a2 100644 --- a/src/TensorFlowNET.Keras/Layers/Merging/Concatenate.cs +++ b/src/TensorFlowNET.Keras/Layers/Merging/Concatenate.cs @@ -37,6 +37,7 @@ public override void build(Shape input_shape) }).ToArray(); shape_set.Add(shape); }*/ + _buildInputShape = input_shape; } protected override Tensors _merge_function(Tensors inputs) diff --git a/src/TensorFlowNET.Keras/Layers/Merging/Merge.cs b/src/TensorFlowNET.Keras/Layers/Merging/Merge.cs index 0363d58f4..3cd43af92 100644 --- a/src/TensorFlowNET.Keras/Layers/Merging/Merge.cs +++ b/src/TensorFlowNET.Keras/Layers/Merging/Merge.cs @@ -17,6 +17,7 @@ public Merge(MergeArgs args) : base(args) public override void build(Shape input_shape) { // output_shape = input_shape.dims[1^]; + _buildInputShape = input_shape; } protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) diff --git a/src/TensorFlowNET.Keras/Layers/Normalization/BatchNormalization.cs b/src/TensorFlowNET.Keras/Layers/Normalization/BatchNormalization.cs index dac92f812..3b8e1ee8d 100644 --- a/src/TensorFlowNET.Keras/Layers/Normalization/BatchNormalization.cs +++ b/src/TensorFlowNET.Keras/Layers/Normalization/BatchNormalization.cs @@ -58,7 +58,7 @@ public override void build(Shape input_shape) var ndims = input_shape.ndim; foreach (var (idx, x) in enumerate(axis)) if (x < 0) - axis[idx] = ndims + x; + args.Axis.dims[idx] = axis[idx] = ndims + x; fused = ndims == 4; @@ -118,6 +118,7 @@ public override void build(Shape input_shape) throw new NotImplementedException("build when renorm is true"); built = true; + _buildInputShape = input_shape; } public override Shape ComputeOutputShape(Shape input_shape) diff --git a/src/TensorFlowNET.Keras/Layers/Normalization/LayerNormalization.cs b/src/TensorFlowNET.Keras/Layers/Normalization/LayerNormalization.cs index 5eebd7350..e19b9c30e 100644 --- a/src/TensorFlowNET.Keras/Layers/Normalization/LayerNormalization.cs +++ b/src/TensorFlowNET.Keras/Layers/Normalization/LayerNormalization.cs @@ -81,6 +81,7 @@ public override void build(Shape input_shape) _fused = _fused_can_be_used(ndims); built = true; + _buildInputShape = input_shape; } bool _fused_can_be_used(int ndims) diff --git a/src/TensorFlowNET.Keras/Layers/Rescaling/Rescaling.cs b/src/TensorFlowNET.Keras/Layers/Preprocessing/Rescaling.cs similarity index 100% rename from src/TensorFlowNET.Keras/Layers/Rescaling/Rescaling.cs rename to src/TensorFlowNET.Keras/Layers/Preprocessing/Rescaling.cs diff --git a/src/TensorFlowNET.Keras/Layers/Cropping/Cropping1D.cs b/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping1D.cs similarity index 77% rename from src/TensorFlowNET.Keras/Layers/Cropping/Cropping1D.cs rename to src/TensorFlowNET.Keras/Layers/Reshaping/Cropping1D.cs index 45f5bf0f6..10c15b698 100644 --- a/src/TensorFlowNET.Keras/Layers/Cropping/Cropping1D.cs +++ b/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping1D.cs @@ -1,11 +1,12 @@ -using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.ArgsDefinition.Reshaping; using Tensorflow.Keras.Engine; -namespace Tensorflow.Keras.Layers { +namespace Tensorflow.Keras.Layers.Reshaping +{ public class Cropping1D : Layer { - CroppingArgs args; - public Cropping1D(CroppingArgs args) : base(args) + Cropping1DArgs args; + public Cropping1D(Cropping1DArgs args) : base(args) { this.args = args; } @@ -22,6 +23,7 @@ public override void build(Shape input_shape) throw new ValueError("The `cropping` argument must be a tuple of 2 integers."); } built = true; + _buildInputShape = input_shape; } protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) @@ -40,7 +42,7 @@ protected override Tensors Call(Tensors inputs, Tensor state = null, bool? train else { int crop_start = args.cropping[0], crop_end = args.cropping[1]; - output = output[new Slice(), new Slice(crop_start, (int)(output.shape[1]) - crop_end), new Slice()]; + output = output[new Slice(), new Slice(crop_start, (int)output.shape[1] - crop_end), new Slice()]; } return output; } @@ -50,12 +52,12 @@ public override Shape ComputeOutputShape(Shape input_shape) if (args.cropping.shape[0] == 1) { int crop = args.cropping[0]; - return new Shape((int)(input_shape[0]), (int)(input_shape[1] - crop * 2), (int)(input_shape[2])); + return new Shape((int)input_shape[0], (int)(input_shape[1] - crop * 2), (int)input_shape[2]); } else { int crop_start = args.cropping[0], crop_end = args.cropping[1]; - return new Shape((int)(input_shape[0]), (int)(input_shape[1] - crop_start - crop_end), (int)(input_shape[2])); + return new Shape((int)input_shape[0], (int)(input_shape[1] - crop_start - crop_end), (int)input_shape[2]); } } } diff --git a/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping2D.cs b/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping2D.cs new file mode 100644 index 000000000..a8d7043ed --- /dev/null +++ b/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping2D.cs @@ -0,0 +1,140 @@ +using Tensorflow.Keras.ArgsDefinition.Reshaping; +using Tensorflow.Keras.Engine; + +namespace Tensorflow.Keras.Layers.Reshaping +{ + /// + /// Crop the input along axis 1 and 2. + /// For example: + /// shape (1, 5, 5, 5) -- crop2D ((1, 2), (1, 3)) --> shape (1, 2, 1, 5) + /// + public class Cropping2D : Layer + { + Cropping2DArgs args; + public Cropping2D(Cropping2DArgs args) : base(args) + { + this.args = args; + } + public override void build(Shape input_shape) + { + built = true; + _buildInputShape = input_shape; + } + protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + { + Tensor output = inputs; + if (output.rank != 4) + { + // throw an ValueError exception + throw new ValueError("Expected dim=4, found dim=" + output.rank); + } + if (args.cropping.shape == new Shape(1)) + { + int crop = args.cropping[0]; + if (args.data_format == Cropping2DArgs.DataFormat.channels_last) + { + output = output[new Slice(), + new Slice(crop, (int)output.shape[1] - crop), + new Slice(crop, (int)output.shape[2] - crop), + new Slice()]; + } + else + { + output = output[new Slice(), + new Slice(), + new Slice(crop, (int)output.shape[2] - crop), + new Slice(crop, (int)output.shape[3] - crop)]; + } + } + // a tuple of 2 integers + else if (args.cropping.shape == new Shape(2)) + { + int crop_1 = args.cropping[0]; + int crop_2 = args.cropping[1]; + if (args.data_format == Cropping2DArgs.DataFormat.channels_last) + { + output = output[new Slice(), + new Slice(crop_1, (int)output.shape[1] - crop_1), + new Slice(crop_2, (int)output.shape[2] - crop_2), + new Slice()]; + } + else + { + output = output[new Slice(), + new Slice(), + new Slice(crop_1, (int)output.shape[2] - crop_1), + new Slice(crop_2, (int)output.shape[3] - crop_2)]; + } + } + else if (args.cropping.shape[0] == 2 && args.cropping.shape[1] == 2) + { + int x_start = args.cropping[0, 0], x_end = args.cropping[0, 1]; + int y_start = args.cropping[1, 0], y_end = args.cropping[1, 1]; + if (args.data_format == Cropping2DArgs.DataFormat.channels_last) + { + output = output[new Slice(), + new Slice(x_start, (int)output.shape[1] - x_end), + new Slice(y_start, (int)output.shape[2] - y_end), + new Slice()]; + } + else + { + output = output[new Slice(), + new Slice(), + new Slice(x_start, (int)output.shape[2] - x_end), + new Slice(y_start, (int)output.shape[3] - y_end) + ]; + } + } + return output; + } + + public override Shape ComputeOutputShape(Shape input_shape) + { + if (args.cropping.shape == new Shape(1)) + { + int crop = args.cropping[0]; + if (args.data_format == Cropping2DArgs.DataFormat.channels_last) + { + return new Shape((int)input_shape[0], (int)input_shape[1] - crop * 2, (int)input_shape[2] - crop * 2, (int)input_shape[3]); + } + else + { + return new Shape((int)input_shape[0], (int)input_shape[1], (int)input_shape[2] - crop * 2, (int)input_shape[3] - crop * 2); + } + } + // a tuple of 2 integers + else if (args.cropping.shape == new Shape(2)) + { + int crop_1 = args.cropping[0], crop_2 = args.cropping[1]; + if (args.data_format == Cropping2DArgs.DataFormat.channels_last) + { + return new Shape((int)input_shape[0], (int)input_shape[1] - crop_1 * 2, (int)input_shape[2] - crop_2 * 2, (int)input_shape[3]); + } + else + { + return new Shape((int)input_shape[0], (int)input_shape[1], (int)input_shape[2] - crop_1 * 2, (int)input_shape[3] - crop_2 * 2); + } + } + else if (args.cropping.shape == new Shape(2, 2)) + { + int crop_1_start = args.cropping[0, 0], crop_1_end = args.cropping[0, 1]; + int crop_2_start = args.cropping[1, 0], crop_2_end = args.cropping[1, 1]; + if (args.data_format == Cropping2DArgs.DataFormat.channels_last) + { + return new Shape((int)input_shape[0], (int)input_shape[1] - crop_1_start - crop_1_end, + (int)input_shape[2] - crop_2_start - crop_2_end, (int)input_shape[3]); + } + else + { + return new Shape((int)input_shape[0], (int)input_shape[1], + (int)input_shape[2] - crop_1_start - crop_1_end, (int)input_shape[3] - crop_2_start - crop_2_end); + } + } + else + { + throw new ValueError(); + } + } + } +} diff --git a/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping3D.cs b/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping3D.cs new file mode 100644 index 000000000..796c2dd33 --- /dev/null +++ b/src/TensorFlowNET.Keras/Layers/Reshaping/Cropping3D.cs @@ -0,0 +1,150 @@ +using Tensorflow.Keras.ArgsDefinition.Reshaping; +using Tensorflow.Keras.Engine; + +namespace Tensorflow.Keras.Layers.Reshaping +{ + /// + /// Similar to copping 2D + /// + public class Cropping3D : Layer + { + Cropping3DArgs args; + public Cropping3D(Cropping3DArgs args) : base(args) + { + this.args = args; + } + + public override void build(Shape input_shape) + { + built = true; + _buildInputShape = input_shape; + } + + protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + { + Tensor output = inputs; + if (output.rank != 5) + { + // throw an ValueError exception + throw new ValueError("Expected dim=5, found dim=" + output.rank); + } + + if (args.cropping.shape == new Shape(1)) + { + int crop = args.cropping[0]; + if (args.data_format == Cropping3DArgs.DataFormat.channels_last) + { + output = output[new Slice(), + new Slice(crop, (int)output.shape[1] - crop), + new Slice(crop, (int)output.shape[2] - crop), + new Slice(crop, (int)output.shape[3] - crop), + new Slice()]; + } + else + { + output = output[new Slice(), + new Slice(), + new Slice(crop, (int)output.shape[2] - crop), + new Slice(crop, (int)output.shape[3] - crop), + new Slice(crop, (int)output.shape[4] - crop)]; + } + + } + // int[1][3] equivalent to a tuple of 3 integers + else if (args.cropping.shape == new Shape(3)) + { + var crop_1 = args.cropping[0]; + var crop_2 = args.cropping[1]; + var crop_3 = args.cropping[2]; + if (args.data_format == Cropping3DArgs.DataFormat.channels_last) + { + output = output[new Slice(), + new Slice(crop_1, (int)output.shape[1] - crop_1), + new Slice(crop_2, (int)output.shape[2] - crop_2), + new Slice(crop_3, (int)output.shape[3] - crop_3), + new Slice()]; + } + else + { + output = output[new Slice(), + new Slice(), + new Slice(crop_1, (int)output.shape[2] - crop_1), + new Slice(crop_2, (int)output.shape[3] - crop_2), + new Slice(crop_3, (int)output.shape[4] - crop_3)]; + } + } + else if (args.cropping.shape[0] == 3 && args.cropping.shape[1] == 2) + { + int x = args.cropping[0, 0], x_end = args.cropping[0, 1]; + int y = args.cropping[1, 0], y_end = args.cropping[1, 1]; + int z = args.cropping[2, 0], z_end = args.cropping[2, 1]; + if (args.data_format == Cropping3DArgs.DataFormat.channels_last) + { + output = output[new Slice(), + new Slice(x, (int)output.shape[1] - x_end), + new Slice(y, (int)output.shape[2] - y_end), + new Slice(z, (int)output.shape[3] - z_end), + new Slice()]; + } + else + { + output = output[new Slice(), + new Slice(), + new Slice(x, (int)output.shape[2] - x_end), + new Slice(y, (int)output.shape[3] - y_end), + new Slice(z, (int)output.shape[4] - z_end) + ]; + } + } + return output; + } + public override Shape ComputeOutputShape(Shape input_shape) + { + if (args.cropping.shape == new Shape(1)) + { + int crop = args.cropping[0]; + if (args.data_format == Cropping3DArgs.DataFormat.channels_last) + { + return new Shape((int)input_shape[0], (int)input_shape[1] - crop * 2, (int)input_shape[2] - crop * 2, (int)input_shape[3] - crop * 2, (int)input_shape[4]); + } + else + { + return new Shape((int)input_shape[0], (int)input_shape[1], (int)input_shape[2] - crop * 2, (int)input_shape[3] - crop * 2, (int)input_shape[4] - crop * 2); + } + } + // int[1][3] equivalent to a tuple of 3 integers + else if (args.cropping.shape == new Shape(3)) + { + var crop_start_1 = args.cropping[0]; + var crop_start_2 = args.cropping[1]; + var crop_start_3 = args.cropping[2]; + if (args.data_format == Cropping3DArgs.DataFormat.channels_last) + { + return new Shape((int)input_shape[0], (int)input_shape[1] - crop_start_1 * 2, (int)input_shape[2] - crop_start_2 * 2, (int)input_shape[3] - crop_start_3 * 2, (int)input_shape[4]); + } + else + { + return new Shape((int)input_shape[0], (int)input_shape[1], (int)input_shape[2] - crop_start_1 * 2, (int)input_shape[3] - crop_start_2 * 2, (int)input_shape[4] - crop_start_3 * 2); + } + } + else if (args.cropping.shape == new Shape(3, 2)) + { + int x = args.cropping[0, 0], x_end = args.cropping[0, 1]; + int y = args.cropping[1, 0], y_end = args.cropping[1, 1]; + int z = args.cropping[2, 0], z_end = args.cropping[2, 1]; + if (args.data_format == Cropping3DArgs.DataFormat.channels_last) + { + return new Shape((int)input_shape[0], (int)input_shape[1] - x - x_end, (int)input_shape[2] - y - y_end, (int)input_shape[3] - z - z_end, (int)input_shape[4]); + } + else + { + return new Shape((int)input_shape[0], (int)input_shape[1], (int)input_shape[2] - x - x_end, (int)input_shape[3] - y - y_end, (int)input_shape[4] - z - z_end); + } + } + else + { + throw new ValueError(); + } + } + } +} diff --git a/src/TensorFlowNET.Keras/Layers/Reshaping/Permute.cs b/src/TensorFlowNET.Keras/Layers/Reshaping/Permute.cs index 868506b6b..8e7a19a9a 100644 --- a/src/TensorFlowNET.Keras/Layers/Reshaping/Permute.cs +++ b/src/TensorFlowNET.Keras/Layers/Reshaping/Permute.cs @@ -24,6 +24,7 @@ public override void build(Shape input_shape) permute = new int[input_shape.rank]; dims.CopyTo(permute, 1); built = true; + _buildInputShape = input_shape; } protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) { diff --git a/src/TensorFlowNET.Keras/Layers/Lstm/LSTM.cs b/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs similarity index 87% rename from src/TensorFlowNET.Keras/Layers/Lstm/LSTM.cs rename to src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs index b7d973847..59555e62b 100644 --- a/src/TensorFlowNET.Keras/Layers/Lstm/LSTM.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/LSTM.cs @@ -1,9 +1,8 @@ using System.Linq; -using Tensorflow.Keras.ArgsDefinition.Lstm; +using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; -using Tensorflow.Keras.Layers.Rnn; -namespace Tensorflow.Keras.Layers.Lstm +namespace Tensorflow.Keras.Layers.Rnn { /// /// Long Short-Term Memory layer - Hochreiter 1997. diff --git a/src/TensorFlowNET.Keras/Layers/Lstm/LSTMCell.cs b/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs similarity index 72% rename from src/TensorFlowNET.Keras/Layers/Lstm/LSTMCell.cs rename to src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs index 3cd35a091..a622c91a9 100644 --- a/src/TensorFlowNET.Keras/Layers/Lstm/LSTMCell.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/LSTMCell.cs @@ -1,7 +1,7 @@ -using Tensorflow.Keras.ArgsDefinition.Lstm; +using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; -namespace Tensorflow.Keras.Layers.Lstm +namespace Tensorflow.Keras.Layers.Rnn { public class LSTMCell : Layer { diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs index 877c35994..6b755ecee 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/RNN.cs @@ -3,7 +3,6 @@ using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; -using Tensorflow.Keras.Layers.Lstm; // from tensorflow.python.distribute import distribution_strategy_context as ds_context; namespace Tensorflow.Keras.Layers.Rnn diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs index a3cd002d9..19669b4b9 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/SimpleRNN.cs @@ -12,7 +12,19 @@ public class SimpleRNN : RNN public SimpleRNN(SimpleRNNArgs args) : base(args) { this.args = args; - cell = new SimpleRNNCell(args); + } + + public override void build(Shape input_shape) + { + var input_dim = input_shape[-1]; + _buildInputShape = input_shape; + + kernel = add_weight("kernel", (input_shape[-1], args.Units), + initializer: args.KernelInitializer + //regularizer = self.kernel_regularizer, + //constraint = self.kernel_constraint, + //caching_device = default_caching_device, + ); } } } \ No newline at end of file diff --git a/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs b/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs index eead274a1..20962df1f 100644 --- a/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs +++ b/src/TensorFlowNET.Keras/Layers/Rnn/StackedRNNCells.cs @@ -4,6 +4,7 @@ using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; +using Tensorflow.Keras.Saving; namespace Tensorflow.Keras.Layers.Rnn { @@ -136,7 +137,7 @@ public void build() // self.built = True } - public override LayerArgs get_config() + public override IKerasConfig get_config() { throw new NotImplementedException(); //def get_config(self): diff --git a/src/TensorFlowNET.Keras/Protobuf/SavedMetadata.cs b/src/TensorFlowNET.Keras/Protobuf/SavedMetadata.cs index 61cec6468..f29f2dec3 100644 --- a/src/TensorFlowNET.Keras/Protobuf/SavedMetadata.cs +++ b/src/TensorFlowNET.Keras/Protobuf/SavedMetadata.cs @@ -194,6 +194,18 @@ public SavedObject() { OnConstruction(); } + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public SavedObject(int nodeId, string nodePath, + global::ThirdParty.Tensorflow.Python.Keras.Protobuf.VersionDef version, string identifier, string metadata) + { + OnConstruction(); + nodeId_ = nodeId; + nodePath_ = nodePath; + identifier_ = identifier; + metadata_ = metadata; + version_ = version; + } + partial void OnConstruction(); [global::System.Diagnostics.DebuggerNonUserCodeAttribute] diff --git a/src/TensorFlowNET.Keras/Protobuf/Versions.cs b/src/TensorFlowNET.Keras/Protobuf/Versions.cs index 40405a5a6..ff9a23c62 100644 --- a/src/TensorFlowNET.Keras/Protobuf/Versions.cs +++ b/src/TensorFlowNET.Keras/Protobuf/Versions.cs @@ -74,6 +74,13 @@ public sealed partial class VersionDef : pb::IMessage { public VersionDef() { OnConstruction(); } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public VersionDef(int producer, int minConsumer) { + OnConstruction(); + producer_ = producer; + minConsumer_ = minConsumer; + } partial void OnConstruction(); diff --git a/src/TensorFlowNET.Keras/Saving/SavedModel/Constants.cs b/src/TensorFlowNET.Keras/Saving/SavedModel/Constants.cs new file mode 100644 index 000000000..3ea4f067e --- /dev/null +++ b/src/TensorFlowNET.Keras/Saving/SavedModel/Constants.cs @@ -0,0 +1,41 @@ +using System.Collections.Generic; + +namespace Tensorflow.Keras.Saving.SavedModel; + +public static class Constants +{ + /// + /// Namespace used to store all attributes added during serialization. + /// e.g. the list of layers can be accessed using `loaded.keras_api.layers`, in an + /// object loaded from `tf.saved_model.load()`. + /// + public static readonly string KERAS_ATTR = "keras_api"; + /// + /// Keys for the serialization cache. + /// Maps to the keras serialization dict {Layer --> SerializedAttributes object} + /// + public static readonly string KERAS_CACHE_KEY = "keras_serialized_attributes"; + /// + /// Name of Keras metadata file stored in the SavedModel. + /// + public static readonly string SAVED_METADATA_PATH = "keras_metadata.pb"; + + public static readonly string INPUT_LAYER_IDENTIFIER = "_tf_keras_input_layer"; + public static readonly string LAYER_IDENTIFIER = "_tf_keras_layer"; + public static readonly string METRIC_IDENTIFIER = "_tf_keras_metric"; + public static readonly string MODEL_IDENTIFIER = "_tf_keras_model"; + public static readonly string NETWORK_IDENTIFIER = "_tf_keras_network"; + public static readonly string RNN_LAYER_IDENTIFIER = "_tf_keras_rnn_layer"; + public static readonly string SEQUENTIAL_IDENTIFIER = "_tf_keras_sequential"; + + public static readonly IList KERAS_OBJECT_IDENTIFIERS = new List() + { + INPUT_LAYER_IDENTIFIER, + LAYER_IDENTIFIER, + METRIC_IDENTIFIER, + MODEL_IDENTIFIER, + NETWORK_IDENTIFIER, + RNN_LAYER_IDENTIFIER, + SEQUENTIAL_IDENTIFIER + }; +} diff --git a/src/TensorFlowNET.Keras/Saving/SavedModel/Save.cs b/src/TensorFlowNET.Keras/Saving/SavedModel/Save.cs new file mode 100644 index 000000000..c7b7e52f4 --- /dev/null +++ b/src/TensorFlowNET.Keras/Saving/SavedModel/Save.cs @@ -0,0 +1,162 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Google.Protobuf; +using Tensorflow.Functions; +using Tensorflow.Keras.Engine; +using Tensorflow.ModelSaving; +using Tensorflow.Train; +using Tensorflow.Keras.Optimizers; +using ThirdParty.Tensorflow.Python.Keras.Protobuf; +using static Tensorflow.Binding; +using Tensorflow.Training; + + +namespace Tensorflow.Keras.Saving.SavedModel; + +public partial class KerasSavedModelUtils +{ + public static void Save(Model model, string filepath, bool overwrite, bool include_optimizer, ConcreteFunction? signatures, + SaveOptions? options, bool save_traces = true) + { + if (!overwrite && File.Exists(filepath)) + { + throw new Exception("The file already exists but is not allowed to overwrite it."); + } + + if (save_traces) + { + if(should_skip_serialization(model)) + { + throw new NotImplementedException(); + } + } + + OptimizerV2? orig_optimizer = null; + if (!include_optimizer) + { + orig_optimizer = model.Optimizer; + model.Optimizer = null; + model._delete_tracking("optimizer"); + } + + IList saved_nodes; + IDictionary> node_paths; + // skip two scopes of python + using (KerasSavedModelUtils.keras_option_scope(save_traces)) + { + (saved_nodes, node_paths) = Tensorflow.SavedModelUtils.save_and_return_nodes(model, filepath, signatures, options); + } + + var metadata = generate_keras_metadata(saved_nodes, node_paths); + File.WriteAllBytes(Path.Combine(filepath, Constants.SAVED_METADATA_PATH), metadata.ToByteArray()); + //File.WriteAllText(Path.Combine(filepath, Constants.SAVED_METADATA_PATH), metadata.ToString()); + + if (!include_optimizer) + { + model.Optimizer = orig_optimizer!; + } + } + + public static SavedMetadata generate_keras_metadata(IList saved_nodes, + IDictionary> node_paths) + { + var metadata = new SavedMetadata(); + for (int i = 0; i < saved_nodes.Count; i++) + { + var node = saved_nodes[i]; + if (node is not Layer) + { + continue; + } + + Layer layer = (Layer)node; + + var path = node_paths[node]; + string node_path; + if (path is null || path.Count() == 0) + { + node_path = "root"; + } + else + { + node_path = $"root.{string.Join(".", path.Select(x => x.Name))}"; + } + + ThirdParty.Tensorflow.Python.Keras.Protobuf.SavedObject saved_object = new() + { + NodeId = i, + NodePath = node_path, + Version = new ThirdParty.Tensorflow.Python.Keras.Protobuf.VersionDef() + { + Producer = 2, + MinConsumer = 1, + BadConsumers = { } + }, + Identifier = layer.ObjectIdentifier, + Metadata = layer.TrackingMetadata + }; + + metadata.Nodes.Add(saved_object); + } + + return metadata; + } + + public static bool should_skip_serialization(object layer) + { + return false; + } + + /// + /// Returns extra trackable objects to attach to the serialized layer. + /// + /// + /// + /// + public static IDictionary wrap_layer_objects(Layer layer, IDictionary> serialization_cache) + { + // TODO: deal with losses and metrics. Currently, `Layer` lacks these two APIs. + + // TODO: change the inherits of `Variable` and revise the implmentation. + var variables = TrackableDataStructure.wrap_or_unwrap(layer.Variables.Select(x => + { + if (x is ResourceVariable or RefVariable) return (Trackable)x; + else throw new TypeError($"The type{x.GetType()} is not supported for the wrapping of layer."); + })); + var trainable_variables = TrackableDataStructure.wrap_or_unwrap(layer.TrainableVariables.Select(x => + { + if (x is ResourceVariable or RefVariable) return (Trackable)x; + else throw new TypeError($"The type{x.GetType()} is not supported for the wrapping of layer."); + })); + var non_trainable_variables = TrackableDataStructure.wrap_or_unwrap(layer.non_trainable_variables.Select(x => + { + if (x is ResourceVariable or RefVariable) return (Trackable)x; + else throw new TypeError($"The type{x.GetType()} is not supported for the wrapping of layer."); + })); + + Dictionary res = new(); + res["variables"] = variables; + res["trainable_variables"] = trainable_variables; + res["non_trainable_variables"] = non_trainable_variables; + res["layers"] = TrackableDataStructure.wrap_or_unwrap(KerasSavedModelUtils.list_all_layers(layer).Select(x => x.GetTrackable())); + + return res; + } + + /// + /// Returns dict of wrapped layer call function and losses in tf.functions. + /// + /// + /// + /// + public static IDictionary wrap_layer_functions(Layer layer, IDictionary> serialization_cache) + { + // TODO: deal with type `RevivedLayer` and `Sequential`. + + // skip the process because of lack of APIs of `Layer`. + + return new Dictionary(); + } +} diff --git a/src/TensorFlowNET.Keras/Saving/SavedModel/base_serialization.cs b/src/TensorFlowNET.Keras/Saving/SavedModel/base_serialization.cs new file mode 100644 index 000000000..eb88c8953 --- /dev/null +++ b/src/TensorFlowNET.Keras/Saving/SavedModel/base_serialization.cs @@ -0,0 +1,37 @@ +using System.Collections.Generic; +using System.Linq; +using Tensorflow.Keras.Engine; +using Newtonsoft.Json; +using Tensorflow.Train; + +namespace Tensorflow.Keras.Saving.SavedModel; + +public abstract class SavedModelSaver +{ + protected Trackable _obj; + public SavedModelSaver(Trackable obj) + { + _obj = obj; + } + + public abstract string ObjectIdentifier { get; } + public abstract string TrackingMetadata { get; } + + public abstract IDictionary objects_to_serialize( + IDictionary> serialization_cache); + + public abstract IDictionary functions_to_serialize( + IDictionary> serialization_cache); + + public IDictionary trackable_children(IDictionary> serialization_cache) + { + if (!KerasSavedModelUtils.ShouldHaveTraces) + { + return new Dictionary(); + } + + var children = objects_to_serialize(serialization_cache); + return children.Concat(functions_to_serialize(serialization_cache).ToDictionary(x => x.Key, x => (Trackable)x.Value)) + .ToDictionary(x => x.Key, x => x.Value); + } +} diff --git a/src/TensorFlowNET.Keras/Saving/SavedModel/layer_serialization.cs b/src/TensorFlowNET.Keras/Saving/SavedModel/layer_serialization.cs new file mode 100644 index 000000000..03693cb57 --- /dev/null +++ b/src/TensorFlowNET.Keras/Saving/SavedModel/layer_serialization.cs @@ -0,0 +1,165 @@ +using System.Collections.Generic; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.Engine; +using Tensorflow.Keras.Layers; +using Tensorflow.Keras.Utils; +using Tensorflow.Train; + +namespace Tensorflow.Keras.Saving.SavedModel; + +public class LayerSavedModelSaver: SavedModelSaver +{ + private Layer _layer; + public LayerSavedModelSaver(Layer obj): base(obj) + { + _obj = obj; + _layer = obj; + } + public override string ObjectIdentifier + { + get => Constants.LAYER_IDENTIFIER; + } + + public override IDictionary objects_to_serialize(IDictionary> serialization_cache) + { + return get_serialized_attributes(serialization_cache).ObjectsToSerialize; + } + + public override IDictionary functions_to_serialize(IDictionary> serialization_cache) + { + return get_serialized_attributes(serialization_cache).FunctionsToSerialize; + } + + /// + /// Generates or retrieves serialized attributes from cache. + /// + /// + protected ISerializedAttributes get_serialized_attributes(IDictionary> serialization_cache) + { + // TODO: deal with cache. + IDictionary keras_cache; + if(serialization_cache is not null && serialization_cache.ContainsKey(Constants.KERAS_CACHE_KEY)) + { + keras_cache = serialization_cache[Constants.KERAS_CACHE_KEY]; + } + else + { + serialization_cache![Constants.KERAS_CACHE_KEY] = keras_cache = new Dictionary(); + } + if (keras_cache.ContainsKey(_obj)) return keras_cache[_obj]; + + var serialized_attr = keras_cache[_obj] = SerializedAttributes.Create(_obj); + + // TODO: complete the statement. Currently the `Layer` lacks member `_must_restore_from_config`. + if (KerasSavedModelUtils.should_skip_serialization(_obj)) + { + return serialized_attr; + } + + var (object_dict, function_dict) = get_serialized_attributes_internal(serialization_cache); + + serialized_attr.set_and_validate_objects(object_dict); + serialized_attr.set_and_validate_functions(function_dict); + return serialized_attr; + } + + /// + /// Returns dictionary of serialized attributes. + /// + /// + private (IDictionary, IDictionary) get_serialized_attributes_internal(IDictionary> serialization_cache) + { + var objects = KerasSavedModelUtils.wrap_layer_objects(_layer, serialization_cache); + var functions = KerasSavedModelUtils.wrap_layer_functions(_layer, serialization_cache); + + functions["_default_save_signature"] = null; + + return (objects, functions); + } + + public override string TrackingMetadata + { + get + { + JObject metadata = new JObject(); + metadata["name"] = _layer.Name; + metadata["trainable"] = _layer.Trainable; + // TODO: implement `expects_training_arg`. + metadata["expects_training_arg"] = false; + metadata["dtype"] = _layer.DType.as_python_name(); + metadata["batch_input_shape"] = _layer.BatchInputShape is null ? null : JToken.FromObject(_layer.BatchInputShape); + // metadata["stateful"] = _obj.stateful; + // metadata["must_restore_from_config"] = _obj.must_restore_from_config; + // metadata["preserve_input_structure_in_config"] = _obj.preserve_input_structure_in_config; + metadata["autocast"] = _layer.AutoCast; + + if(_layer.InputSpec is not null) + { + metadata["input_spec"] = generic_utils.serialize_keras_object(_layer.InputSpec); + } + + metadata.Merge(get_serialized(_layer), new JsonMergeSettings + { + // Handle conflicts by using values from obj2 + MergeArrayHandling = MergeArrayHandling.Merge + }); + // skip the check of `input_spec` and `build_input_shape` for the lack of members. + // skip the check of `activity_regularizer` for the type problem. + if(_layer.BuildInputShape is not null) + { + metadata["build_input_shape"] = JToken.FromObject(_layer.BuildInputShape); + } + return metadata.ToString(); + } + } + + public static JObject get_serialized(Layer obj) + { + return generic_utils.serialize_keras_object(obj); + } +} + +public class InputLayerSavedModelSaver: SavedModelSaver +{ + public InputLayerSavedModelSaver(Layer obj) : base(obj) + { + + } + public override string ObjectIdentifier => Constants.INPUT_LAYER_IDENTIFIER; + + public override IDictionary functions_to_serialize(IDictionary> serialization_cache) + { + return new Dictionary(); + } + + public override IDictionary objects_to_serialize(IDictionary> serialization_cache) + { + return new Dictionary(); + } + + public override string TrackingMetadata + { + get + { + if(_obj is not InputLayer) + { + throw new TypeError($"The type {_obj.GetType()} cannot be recognized as an input layer."); + } + var layer = (InputLayer)_obj; + var config = (layer.get_config() as InputLayerArgs)!; + var info = new + { + class_name = layer.GetType().Name, + name = layer.Name, + dtype = layer.DType, + sparse = config.Sparse, + ragged = config.Ragged, + batch_input_shape = layer.BatchInputShape, + config = layer.get_config() + }; + return JsonConvert.SerializeObject(info); + } + } +} diff --git a/src/TensorFlowNET.Keras/Saving/SavedModel/serialized_attributes.cs b/src/TensorFlowNET.Keras/Saving/SavedModel/serialized_attributes.cs new file mode 100644 index 000000000..ac194c00f --- /dev/null +++ b/src/TensorFlowNET.Keras/Saving/SavedModel/serialized_attributes.cs @@ -0,0 +1,282 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Tensorflow.Keras.Engine; +using Tensorflow.Keras.Layers.Rnn; +using Tensorflow.Keras.Metrics; +using Tensorflow.Train; + +namespace Tensorflow.Keras.Saving.SavedModel +{ + // TODO: revise the name of these "Attributes". Since "Attribute" is a significant feature of C#, + // Using the name "Attributes" may be quite confusing. + /// + /// Class that tracks and validates all serialization attributes. + /// + public abstract class SerializedAttributes: ISerializedAttributes + { + protected IDictionary _object_dict; + protected IDictionary _function_dict; + protected AutoTrackable _keras_trackable; + protected HashSet _all_functions; + protected HashSet _all_checkpointable_objects; + + private SerializedAttributes() + { + _object_dict= new Dictionary(); + _function_dict= new Dictionary(); + _keras_trackable= new AutoTrackable(); + _all_functions= new HashSet(); + _all_checkpointable_objects= new HashSet(); + } + + protected SerializedAttributes(IEnumerable checkpointable_objects, IEnumerable functions) + { + _object_dict = new Dictionary(); + _function_dict = new Dictionary(); + _keras_trackable = new AutoTrackable(); + + _all_checkpointable_objects = new HashSet(checkpointable_objects); + _all_functions = new HashSet(functions); + } + + protected SerializedAttributes((IEnumerable, IEnumerable) objects_and_functions) + { + _object_dict = new Dictionary(); + _function_dict = new Dictionary(); + _keras_trackable = new AutoTrackable(); + + _all_checkpointable_objects = new HashSet(objects_and_functions.Item1); + _all_functions = new HashSet(objects_and_functions.Item2); + } + + public IDictionary Functions => _function_dict.TakeWhile(x => x.Value is not null).ToDictionary(x => x.Key, x => x.Value!); + + public IDictionary CheckpointableObjects => _object_dict.TakeWhile(x => x.Value is not null).ToDictionary(x => x.Key, x => x.Value!); + + /// + /// Returns functions to attach to the root object during serialization. + /// + public IDictionary FunctionsToSerialize + { + get + { + Dictionary functions = new(); + foreach(var pair in Functions) + { + if (_all_functions.Contains(pair.Key)) + { + // TODO: deal with `LayerCall`. + functions[pair.Key] = pair.Value; + } + } + return functions; + } + } + + /// + /// Returns objects to attach to the root object during serialization. + /// + public IDictionary ObjectsToSerialize + { + get + { + var objects = CheckpointableObjects.TakeWhile( x=> _all_checkpointable_objects.Contains(x.Key)).ToDictionary(x => x.Key, x => x.Value); + objects[Constants.KERAS_ATTR] = _keras_trackable; + return objects; + } + } + + /// + /// Saves function dictionary, and validates dictionary values. + /// + /// + public IDictionary set_and_validate_functions(IDictionary function_dict) + { + foreach(var key in _all_functions) + { + if (function_dict.ContainsKey(key)) + { + // TODO: deal with type `LayerCall`. + var fn = function_dict[key]; + if (fn is not null && (fn is not Function)) + { + throw new ValueError($"Function dictionary contained a non-function object: {function_dict[key]} (for key {key})."); + } + _function_dict[key] = fn; + + var tf_fn = fn; // TODO: deal with type `LayerCall`. + + // Warning: this implmentation should be considered again. + var properties = _keras_trackable.GetType().GetProperties(); + foreach (var property in properties) + { + if(property.Name == key) + { + property.SetValue(_keras_trackable, tf_fn); + break; + } + } + } + else + { + throw new ValueError($"Function {key} missing from serialized function dict."); + } + } + return Functions; + } + + /// + /// Saves objects to a dictionary, and validates the values. + /// + /// + public IDictionary set_and_validate_objects(IDictionary object_dict) + { + foreach(var key in _all_checkpointable_objects) + { + if (object_dict.ContainsKey(key)) + { + _object_dict[key] = object_dict[key]; + // Warning: this implmentation should be considered again. + var properties = _keras_trackable.GetType().GetProperties(); + foreach (var property in properties) + { + if (property.Name == key) + { + property.SetValue(_keras_trackable, object_dict[key]); + break; + } + } + } + else + { + throw new ValueError($"Object {key} missing from serialized object dict."); + } + } + return CheckpointableObjects; + } + + /// + /// Returns a new SerializedAttribute object (corresponding to `new` of tensorflow python). + /// + /// + public static SerializedAttributes Create(Trackable obj) + { + if(obj is Model) + { + return new ModelAttributes(); + } + else if(obj is Metric) + { + return new MetricAttributes(); + } + else if(obj is RNN) + { + return new RNNAttributes(); + } + else if(obj is Layer) + { + return new LayerAttributes(); + } + else + { + throw new TypeError($"Internal error during serialization: Expected Keras Layer object, got {obj} of type {obj.GetType()}"); + } + } + + protected virtual (IEnumerable, IEnumerable) get_objects_and_functions_recursively(IEnumerable? checkpointable_objects, IEnumerable? functions) + { + return (checkpointable_objects ?? (new List()), functions ?? (new List())); + } + } + + // Note that the current implementation still has some potential risks. + // The tensorflow python says that this class is "Common endpoints shared by all models loadable by Keras". + // However, currently it's just a normal class. + public class CommonEndPoints: SerializedAttributes + { + public CommonEndPoints(IEnumerable checkpointable_objects, IEnumerable functions) : + //base(checkpointable_objects.Concat(new string[] { "variables", "trainable_variables", "regularization_losses" }), + // functions.Concat(new string[] { "__call__", "call_and_return_all_conditional_losses", "_default_save_signature" })) + base(checkpointable_objects.Concat(new string[] { "variables", "trainable_variables"}), + functions.Concat(new string[] { })) + { + + } + + public CommonEndPoints() : + //base(new string[] { "variables", "trainable_variables", "regularization_losses" }, + // new string[] { "__call__", "call_and_return_all_conditional_losses", "_default_save_signature" }) + base(new string[] { "variables", "trainable_variables"}, + new string[] {}) + { + + } + } + + public class LayerAttributes: CommonEndPoints + { + public LayerAttributes(IEnumerable checkpointable_objects, IEnumerable functions) : + //base(checkpointable_objects.Concat(new string[] { "non_trainable_variables", "layers", "metrics", "layer_regularization_losses", "layer_metrics" }), + // functions.Concat(new string[] { "call_and_return_conditional_losses", "activity_regularizer_fn" }) + base(checkpointable_objects.Concat(new string[] { "non_trainable_variables", "layers"}), + functions.Concat(new string[] { })) + { + + } + + public LayerAttributes() : + //base(new string[] { "non_trainable_variables", "layers", "metrics", "layer_regularization_losses", "layer_metrics" }, + // new string[] { "call_and_return_conditional_losses", "activity_regularizer_fn" }) + base(new string[] { "non_trainable_variables", "layers" }, + new string[] { }) + { + + } + } + + public class ModelAttributes: LayerAttributes + { + public ModelAttributes(IEnumerable checkpointable_objects, IEnumerable functions): + base(checkpointable_objects, functions) + { + + } + + public ModelAttributes(): base() + { + + } + } + + public class MetricAttributes : SerializedAttributes + { + public MetricAttributes(IEnumerable checkpointable_objects, IEnumerable functions) : + base(checkpointable_objects.Concat(new string[] { "variables" }), functions) + { + + } + + public MetricAttributes() : + base(new string[] { "variables" }, new string[] {}) + { + + } + } + + public class RNNAttributes: LayerAttributes + { + public RNNAttributes(IEnumerable checkpointable_objects, IEnumerable functions) : + base(checkpointable_objects, functions.Concat(new string[] {"states"})) + { + + } + + public RNNAttributes() : + base(new string[] { }, new string[] { "states" }) + { + + } + } +} diff --git a/src/TensorFlowNET.Keras/Saving/SavedModel/utils.cs b/src/TensorFlowNET.Keras/Saving/SavedModel/utils.cs new file mode 100644 index 000000000..51f8d2c91 --- /dev/null +++ b/src/TensorFlowNET.Keras/Saving/SavedModel/utils.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using Tensorflow.Keras.Engine; + +namespace Tensorflow.Keras.Saving.SavedModel; + +public partial class KerasSavedModelUtils +{ + public static bool ShouldHaveTraces { get; internal set; } = true; + + public static SaveOptionsContext keras_option_scope(bool save_traces) + { + var res = new SaveOptionsContext(ShouldHaveTraces); + ShouldHaveTraces = save_traces; + return res; + } + + public static IEnumerable list_all_layers(Layer layer) + { + if(layer is Model) + { + return (layer as Model).Layers; + } + else + { + return new List(layer._flatten_layers(false, false)); + } + } +} + +/// +/// Implementation of this class is different with that of python. +/// But it could be used with `using` the same as `with` of python. +/// +public class SaveOptionsContext: IDisposable +{ + public bool _old_value; + public SaveOptionsContext(bool old_value) + { + _old_value = old_value; + } + + public void Dispose() + { + KerasSavedModelUtils.ShouldHaveTraces = _old_value; + } +} diff --git a/src/TensorFlowNET.Keras/Saving/TensorShapeConfig.cs b/src/TensorFlowNET.Keras/Saving/TensorShapeConfig.cs deleted file mode 100644 index 4c2ecc0d8..000000000 --- a/src/TensorFlowNET.Keras/Saving/TensorShapeConfig.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; - -namespace Tensorflow.Keras.Saving -{ - public class TensorShapeConfig - { - public string ClassName { get; set; } - public int?[] Items { get; set; } - - public static implicit operator Shape(TensorShapeConfig shape) - => shape == null ? null : new Shape(shape.Items.Select(x => x.HasValue ? x.Value : -1).ToArray()); - } -} diff --git a/src/TensorFlowNET.Keras/Saving/serialization.cs b/src/TensorFlowNET.Keras/Saving/serialization.cs new file mode 100644 index 000000000..d5e46d11c --- /dev/null +++ b/src/TensorFlowNET.Keras/Saving/serialization.cs @@ -0,0 +1,125 @@ +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text; +using Tensorflow.Keras.Saving.SavedModel; + +namespace Tensorflow.Keras.Saving +{ + // TODO: make it thread safe. + public class SharedObjectSavingScope: IDisposable + { + private class WeakReferenceEqualityComparer: IEqualityComparer> + { + public bool Equals(WeakReference x, WeakReference y) + { + if(!x.TryGetTarget(out var tx)) + { + return false; + } + if(!y.TryGetTarget(out var ty)) + { + return false; + } + return tx.Equals(ty); + } + public int GetHashCode(WeakReference obj) + { + if (!obj.TryGetTarget(out var w)) + { + return 0; + } + return w.GetHashCode(); + } + } + private static SharedObjectSavingScope? _instance = null; + private readonly Dictionary, int> _shared_object_ids= new Dictionary, int>(); + private int _currentId = 0; + /// + /// record how many times the scope is nested. + /// + private int _nestedDepth = 0; + private SharedObjectSavingScope() + { + + } + + public static SharedObjectSavingScope Enter() + { + if(_instance is not null) + { + _instance._nestedDepth++; + return _instance; + } + else + { + _instance = new SharedObjectSavingScope(); + _instance._nestedDepth++; + return _instance; + } + } + + public static SharedObjectSavingScope GetScope() + { + return _instance; + } + + public int GetId(object? obj) + { + if(obj is null) + { + return _currentId++; + } + var maybe_key = _shared_object_ids.Keys.SingleOrDefault(x => new WeakReferenceEqualityComparer().Equals(x, new WeakReference(obj))); + if (maybe_key is not null) + { + return _shared_object_ids[maybe_key]; + } + _shared_object_ids[new WeakReference(obj)] = _currentId++; + return _currentId; + } + + public void Dispose() + { + _nestedDepth--; + if(_nestedDepth== 0) + { + _instance = null; + } + } + } + + public static class serialize_utils + { + public static readonly string SHARED_OBJECT_KEY = "shared_object_id"; + /// + /// Returns the serialization of the class with the given config. + /// + /// + /// + /// + /// + /// + public static JObject serialize_keras_class_and_config(string class_name, JToken config, object? obj = null, int? shared_object_id = null) + { + JObject res = new JObject(); + res["class_name"] = class_name; + res["config"] = config; + + if(shared_object_id is not null) + { + res[SHARED_OBJECT_KEY] = shared_object_id!; + } + + var scope = SharedObjectSavingScope.GetScope(); + if(scope is not null && obj is not null) + { + res[SHARED_OBJECT_KEY] = scope.GetId(obj); + } + + return res; + } + } +} diff --git a/src/TensorFlowNET.Keras/Utils/base_layer_utils.cs b/src/TensorFlowNET.Keras/Utils/base_layer_utils.cs index 1e6ce4091..d845f3ca9 100644 --- a/src/TensorFlowNET.Keras/Utils/base_layer_utils.cs +++ b/src/TensorFlowNET.Keras/Utils/base_layer_utils.cs @@ -53,7 +53,7 @@ public static IVariableV1 make_variable(VariableArgs args) } /// - /// Makes a layer name (or arbitrary string) unique within a TensorFlow graph. + /// Makes a layer name (or arbitrary string) unique within a TensorFlow graph. (correponding to `backend.unique_object_name` of python.) /// /// /// diff --git a/src/TensorFlowNET.Keras/Utils/generic_utils.cs b/src/TensorFlowNET.Keras/Utils/generic_utils.cs index c2839cdc7..730a33e3e 100644 --- a/src/TensorFlowNET.Keras/Utils/generic_utils.cs +++ b/src/TensorFlowNET.Keras/Utils/generic_utils.cs @@ -14,24 +14,43 @@ You may obtain a copy of the License at limitations under the License. ******************************************************************************/ +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; using System.Linq; +using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Saving; namespace Tensorflow.Keras.Utils { public class generic_utils { - public static LayerConfig serialize_keras_object(ILayer instance) + /// + /// This method does not have corresponding method in python. It's close to `serialize_keras_object`. + /// + /// + /// + public static LayerConfig serialize_layer_to_config(ILayer instance) { var config = instance.get_config(); + Debug.Assert(config is LayerArgs); return new LayerConfig { - Config = config, + Config = config as LayerArgs, ClassName = instance.GetType().Name }; } + public static JObject serialize_keras_object(IKerasConfigable instance) + { + var config = JToken.FromObject(instance.get_config()); + // TODO: change the class_name to registered name, instead of system class name. + return serialize_utils.serialize_keras_class_and_config(instance.GetType().Name, config, instance); + } + public static string to_snake_case(string name) { return string.Concat(name.Select((x, i) => diff --git a/test/TensorFlowNET.Keras.UnitTest/InitializerTest.cs b/test/TensorFlowNET.Keras.UnitTest/InitializerTest.cs index c811b5643..6950e65fc 100644 --- a/test/TensorFlowNET.Keras.UnitTest/InitializerTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/InitializerTest.cs @@ -6,7 +6,7 @@ using TensorFlowNET.Keras.UnitTest; using static Tensorflow.Binding; -namespace Tensorflow.Keras.UnitTest; +namespace TensorFlowNET.Keras.UnitTest; [TestClass] public class InitializerTest : EagerModeTestBase @@ -15,6 +15,6 @@ public class InitializerTest : EagerModeTestBase public void Orthogonal() { var initializer = tf.keras.initializers.Orthogonal(); - var values = initializer.Apply(new InitializerArgs((2, 2))); + var values = initializer.Apply(new Tensorflow.InitializerArgs((2, 2))); } } diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/ModelSaveTest.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/ModelSaveTest.cs index 0a1098af7..67e8ff797 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/ModelSaveTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/ModelSaveTest.cs @@ -1,6 +1,8 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using Tensorflow.Keras.Engine; +using System.Diagnostics; using static Tensorflow.KerasApi; +using Tensorflow.Keras.Saving; namespace TensorFlowNET.Keras.UnitTest { @@ -15,7 +17,8 @@ public void GetAndFromConfig() { var model = GetFunctionalModel(); var config = model.get_config(); - var new_model = keras.models.from_config(config); + Debug.Assert(config is ModelConfig); + var new_model = keras.models.from_config(config as ModelConfig); Assert.AreEqual(model.Layers.Count, new_model.Layers.Count); } diff --git a/test/TensorFlowNET.Keras.UnitTest/SaveModel/SequentialModelTest.cs b/test/TensorFlowNET.Keras.UnitTest/SaveModel/SequentialModelTest.cs new file mode 100644 index 000000000..269b9c058 --- /dev/null +++ b/test/TensorFlowNET.Keras.UnitTest/SaveModel/SequentialModelTest.cs @@ -0,0 +1,202 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Tensorflow.NumPy; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Tensorflow; +using static Tensorflow.Binding; +using static Tensorflow.KerasApi; +using Tensorflow.Keras; +using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.Engine; +using Tensorflow.Keras.Layers; +using Tensorflow.Keras.Losses; +using Tensorflow.Keras.Metrics; +using Tensorflow.Keras.Optimizers; +using Tensorflow.Operations; +using System.Diagnostics; + +namespace TensorFlowNET.Keras.UnitTest.SaveModel; + +[TestClass] +public class SequentialModelTest +{ + [TestMethod] + public void SimpleModelFromAutoCompile() + { + var inputs = new KerasInterface().Input((28, 28, 1)); + var x = new Flatten(new FlattenArgs()).Apply(inputs); + x = new Dense(new DenseArgs() { Units = 100, Activation = tf.nn.relu }).Apply(x); + x = new LayersApi().Dense(units: 10).Apply(x); + var outputs = new LayersApi().Softmax(axis: 1).Apply(x); + var model = new KerasInterface().Model(inputs, outputs); + + model.compile(new Adam(0.001f), new LossesApi().SparseCategoricalCrossentropy(), new string[] { "accuracy" }); + + var data_loader = new MnistModelLoader(); + var num_epochs = 1; + var batch_size = 50; + + var dataset = data_loader.LoadAsync(new ModelLoadSetting + { + TrainDir = "mnist", + OneHot = false, + ValidationSize = 10000, + }).Result; + + model.fit(dataset.Train.Data, dataset.Train.Labels, batch_size, num_epochs); + + model.save("./pb_simple_compile", save_format: "tf"); + } + + [TestMethod] + public void SimpleModelFromSequential() + { + Model model = KerasApi.keras.Sequential(new List() + { + keras.layers.InputLayer((28, 28, 1)), + keras.layers.Flatten(), + keras.layers.Dense(100, "relu"), + keras.layers.Dense(10), + keras.layers.Softmax(1) + }); + + model.compile(new Adam(0.001f), new LossesApi().SparseCategoricalCrossentropy(), new string[] { "accuracy" }); + + var data_loader = new MnistModelLoader(); + var num_epochs = 1; + var batch_size = 50; + + var dataset = data_loader.LoadAsync(new ModelLoadSetting + { + TrainDir = "mnist", + OneHot = false, + ValidationSize = 50000, + }).Result; + + model.fit(dataset.Train.Data, dataset.Train.Labels, batch_size, num_epochs); + + model.save("./pb_simple_sequential", save_format: "tf"); + } + + [TestMethod] + public void AlexModelFromSequential() + { + Model model = KerasApi.keras.Sequential(new List() + { + keras.layers.InputLayer((227, 227, 3)), + keras.layers.Conv2D(96, (11, 11), (4, 4), activation:"relu", padding:"valid"), + keras.layers.BatchNormalization(), + keras.layers.MaxPooling2D((3, 3), strides:(2, 2)), + + keras.layers.Conv2D(256, (5, 5), (1, 1), "same", activation: "relu"), + keras.layers.BatchNormalization(), + keras.layers.MaxPooling2D((3, 3), (2, 2)), + + keras.layers.Conv2D(384, (3, 3), (1, 1), "same", activation: "relu"), + keras.layers.BatchNormalization(), + + keras.layers.Conv2D(384, (3, 3), (1, 1), "same", activation: "relu"), + keras.layers.BatchNormalization(), + + keras.layers.Conv2D(256, (3, 3), (1, 1), "same", activation: "relu"), + keras.layers.BatchNormalization(), + keras.layers.MaxPooling2D((3, 3), (2, 2)), + + keras.layers.Flatten(), + keras.layers.Dense(4096, activation: "relu"), + keras.layers.Dropout(0.5f), + + keras.layers.Dense(4096, activation: "relu"), + keras.layers.Dropout(0.5f), + + keras.layers.Dense(1000, activation: "linear"), + keras.layers.Softmax(1) + }); + + model.compile(new Adam(0.001f), new LossesApi().SparseCategoricalCrossentropy(from_logits:true), new string[] { "accuracy" }); + + var num_epochs = 1; + var batch_size = 8; + + var dataset = new RandomDataSet(new Shape(227, 227, 3), 16); + + model.fit(dataset.Data, dataset.Labels, batch_size, num_epochs); + + model.save("./pb_alex_sequential", save_format: "tf"); + + // The saved model can be test with the following python code: + #region alexnet_python_code + //import pathlib + //import tensorflow as tf + + //def func(a): + // return -a + + //if __name__ == '__main__': + // model = tf.keras.models.load_model("./pb_alex_sequential") + // model.summary() + + // num_classes = 5 + // batch_size = 128 + // img_height = 227 + // img_width = 227 + // epochs = 100 + + // dataset_url = "https://storage.googleapis.com/download.tensorflow.org/example_images/flower_photos.tgz" + // data_dir = tf.keras.utils.get_file('flower_photos', origin = dataset_url, untar = True) + // data_dir = pathlib.Path(data_dir) + + // train_ds = tf.keras.preprocessing.image_dataset_from_directory( + // data_dir, + // validation_split = 0.2, + // subset = "training", + // seed = 123, + // image_size = (img_height, img_width), + // batch_size = batch_size) + + // val_ds = tf.keras.preprocessing.image_dataset_from_directory( + // data_dir, + // validation_split = 0.2, + // subset = "validation", + // seed = 123, + // image_size = (img_height, img_width), + // batch_size = batch_size) + + + // model.compile(optimizer = 'adam', + // loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits = True), + // metrics =['accuracy']) + + // model.build((None, img_height, img_width, 3)) + + // history = model.fit( + // train_ds, + // validation_data = val_ds, + // epochs = epochs + // ) + #endregion + } + + public class RandomDataSet : DataSetBase + { + private Shape _shape; + + public RandomDataSet(Shape shape, int count) + { + _shape = shape; + Debug.Assert(_shape.ndim == 3); + long[] dims = new long[4]; + dims[0] = count; + for (int i = 1; i < 4; i++) + { + dims[i] = _shape[i - 1]; + } + Shape s = new Shape(dims); + Data = np.random.normal(0, 2, s); + Labels = np.random.uniform(0, 1, (count, 1)); + } + } +} \ No newline at end of file From ec340eeff57c7f9bef8fc21dd94f17889b7453b5 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sat, 4 Feb 2023 12:06:21 -0600 Subject: [PATCH 20/52] np.ones_like and np.zeros_like --- src/TensorFlowNET.Console/SimpleRnnTest.cs | 1 - src/TensorFlowNET.Core/Numpy/Numpy.Creation.cs | 9 ++++++--- src/TensorFlowNET.Core/Sessions/BaseSession.cs | 9 ++++++++- src/python/simple_rnn.py | 18 ++++++++++-------- .../Tensorflow.Keras.UnitTest.csproj | 1 - 5 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/TensorFlowNET.Console/SimpleRnnTest.cs b/src/TensorFlowNET.Console/SimpleRnnTest.cs index da1245172..9769eb655 100644 --- a/src/TensorFlowNET.Console/SimpleRnnTest.cs +++ b/src/TensorFlowNET.Console/SimpleRnnTest.cs @@ -12,7 +12,6 @@ public class SimpleRnnTest { public void Run() { - tf.UseKeras(); var inputs = np.random.random((6, 10, 8)).astype(np.float32); //var simple_rnn = tf.keras.layers.SimpleRNN(4); //var output = simple_rnn.Apply(inputs); // The output has shape `[32, 4]`. diff --git a/src/TensorFlowNET.Core/Numpy/Numpy.Creation.cs b/src/TensorFlowNET.Core/Numpy/Numpy.Creation.cs index 7e6a2b656..9604392c1 100644 --- a/src/TensorFlowNET.Core/Numpy/Numpy.Creation.cs +++ b/src/TensorFlowNET.Core/Numpy/Numpy.Creation.cs @@ -2,7 +2,6 @@ using System.Collections; using System.Collections.Generic; using System.IO; -using System.Numerics; using System.Text; using static Tensorflow.Binding; @@ -103,11 +102,15 @@ public static NDArray ndarray(Shape shape, TF_DataType dtype = TF_DataType.TF_DO public static NDArray ones(Shape shape, TF_DataType dtype = TF_DataType.TF_DOUBLE) => new NDArray(tf.ones(shape, dtype: dtype)); - public static NDArray ones_like(NDArray a, Type dtype = null) - => throw new NotImplementedException(""); + public static NDArray ones_like(NDArray a, TF_DataType dtype = TF_DataType.DtInvalid) + => new NDArray(tf.ones_like(a, dtype: dtype)); [AutoNumPy] public static NDArray zeros(Shape shape, TF_DataType dtype = TF_DataType.TF_DOUBLE) => new NDArray(tf.zeros(shape, dtype: dtype)); + + [AutoNumPy] + public static NDArray zeros_like(NDArray a, TF_DataType dtype = TF_DataType.DtInvalid) + => new NDArray(tf.zeros_like(a, dtype: dtype)); } } diff --git a/src/TensorFlowNET.Core/Sessions/BaseSession.cs b/src/TensorFlowNET.Core/Sessions/BaseSession.cs index 0051a6b33..01ba04077 100644 --- a/src/TensorFlowNET.Core/Sessions/BaseSession.cs +++ b/src/TensorFlowNET.Core/Sessions/BaseSession.cs @@ -291,7 +291,14 @@ private void _extend_graph() protected override void DisposeUnmanagedResources(IntPtr handle) { // c_api.TF_CloseSession(handle, tf.Status.Handle); - c_api.TF_DeleteSession(handle, c_api.TF_NewStatus()); + if (tf.Status == null || tf.Status.Handle.IsInvalid) + { + c_api.TF_DeleteSession(handle, c_api.TF_NewStatus()); + } + else + { + c_api.TF_DeleteSession(handle, tf.Status.Handle); + } } } } diff --git a/src/python/simple_rnn.py b/src/python/simple_rnn.py index 97f9f3f31..c5f3b1f2c 100644 --- a/src/python/simple_rnn.py +++ b/src/python/simple_rnn.py @@ -1,15 +1,17 @@ import numpy as np import tensorflow as tf +import tensorflow.experimental.numpy as tnp # tf.experimental.numpy -inputs = np.random.random([32, 10, 8]).astype(np.float32) -simple_rnn = tf.keras.layers.SimpleRNN(4) +inputs = np.arange(6 * 10 * 8).reshape([6, 10, 8]).astype(np.float32) +# simple_rnn = tf.keras.layers.SimpleRNN(4) -output = simple_rnn(inputs) # The output has shape `[32, 4]`. +# output = simple_rnn(inputs) # The output has shape `[6, 4]`. -simple_rnn = tf.keras.layers.SimpleRNN( - 4, return_sequences=True, return_state=True) +simple_rnn = tf.keras.layers.SimpleRNN(4, return_sequences=True, return_state=True) -# whole_sequence_output has shape `[32, 10, 4]`. -# final_state has shape `[32, 4]`. -whole_sequence_output, final_state = simple_rnn(inputs) \ No newline at end of file +# whole_sequence_output has shape `[6, 10, 4]`. +# final_state has shape `[6, 4]`. +whole_sequence_output, final_state = simple_rnn(inputs) +print(whole_sequence_output) +print(final_state) \ No newline at end of file diff --git a/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj b/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj index fc693b1ef..61e522e6c 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj +++ b/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj @@ -4,7 +4,6 @@ net6.0 false - 11.0 AnyCPU;x64 From 271dcefc15c5f5b5170c00304820458b5cfa8de3 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sun, 5 Feb 2023 12:46:31 -0600 Subject: [PATCH 21/52] fix keras model predict return result. --- .../Sessions/BaseSession.cs | 3 +- .../Callbacks/CallbackList.cs | 20 +++++++ src/TensorFlowNET.Keras/Callbacks/History.cs | 21 +++++++ .../Callbacks/ICallback.cs | 4 ++ .../Callbacks/ProgbarLogger.cs | 24 +++++++- .../Engine/Model.Predict.cs | 59 +++++++++++++++++++ 6 files changed, 128 insertions(+), 3 deletions(-) diff --git a/src/TensorFlowNET.Core/Sessions/BaseSession.cs b/src/TensorFlowNET.Core/Sessions/BaseSession.cs index 01ba04077..095187b9a 100644 --- a/src/TensorFlowNET.Core/Sessions/BaseSession.cs +++ b/src/TensorFlowNET.Core/Sessions/BaseSession.cs @@ -293,7 +293,8 @@ protected override void DisposeUnmanagedResources(IntPtr handle) // c_api.TF_CloseSession(handle, tf.Status.Handle); if (tf.Status == null || tf.Status.Handle.IsInvalid) { - c_api.TF_DeleteSession(handle, c_api.TF_NewStatus()); + using var status = new Status(); + c_api.TF_DeleteSession(handle, status.Handle); } else { diff --git a/src/TensorFlowNET.Keras/Callbacks/CallbackList.cs b/src/TensorFlowNET.Keras/Callbacks/CallbackList.cs index bb3ed6edc..54e3780a7 100644 --- a/src/TensorFlowNET.Keras/Callbacks/CallbackList.cs +++ b/src/TensorFlowNET.Keras/Callbacks/CallbackList.cs @@ -39,5 +39,25 @@ public void on_epoch_end(int epoch, Dictionary epoch_logs) { callbacks.ForEach(x => x.on_epoch_end(epoch, epoch_logs)); } + + public void on_predict_begin() + { + callbacks.ForEach(x => x.on_predict_begin()); + } + + public void on_predict_batch_begin(long step) + { + callbacks.ForEach(x => x.on_predict_batch_begin(step)); + } + + public void on_predict_batch_end(long end_step, Dictionary logs) + { + callbacks.ForEach(x => x.on_predict_batch_end(end_step, logs)); + } + + public void on_predict_end() + { + callbacks.ForEach(x => x.on_predict_end()); + } } } diff --git a/src/TensorFlowNET.Keras/Callbacks/History.cs b/src/TensorFlowNET.Keras/Callbacks/History.cs index 02588b5e7..89e1834bc 100644 --- a/src/TensorFlowNET.Keras/Callbacks/History.cs +++ b/src/TensorFlowNET.Keras/Callbacks/History.cs @@ -48,5 +48,26 @@ public void on_epoch_end(int epoch, Dictionary epoch_logs) history[log.Key].Add((float)log.Value); } } + + public void on_predict_begin() + { + epochs = new List(); + history = new Dictionary>(); + } + + public void on_predict_batch_begin(long step) + { + + } + + public void on_predict_batch_end(long end_step, Dictionary logs) + { + + } + + public void on_predict_end() + { + + } } } diff --git a/src/TensorFlowNET.Keras/Callbacks/ICallback.cs b/src/TensorFlowNET.Keras/Callbacks/ICallback.cs index 34763c557..7d71ccace 100644 --- a/src/TensorFlowNET.Keras/Callbacks/ICallback.cs +++ b/src/TensorFlowNET.Keras/Callbacks/ICallback.cs @@ -11,5 +11,9 @@ public interface ICallback void on_train_batch_begin(long step); void on_train_batch_end(long end_step, Dictionary logs); void on_epoch_end(int epoch, Dictionary epoch_logs); + void on_predict_begin(); + void on_predict_batch_begin(long step); + void on_predict_batch_end(long end_step, Dictionary logs); + void on_predict_end(); } } diff --git a/src/TensorFlowNET.Keras/Callbacks/ProgbarLogger.cs b/src/TensorFlowNET.Keras/Callbacks/ProgbarLogger.cs index 17e041014..bb18b2cb3 100644 --- a/src/TensorFlowNET.Keras/Callbacks/ProgbarLogger.cs +++ b/src/TensorFlowNET.Keras/Callbacks/ProgbarLogger.cs @@ -1,5 +1,4 @@ -using PureHDF; -using System; +using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; @@ -77,5 +76,26 @@ void _maybe_init_progbar() { } + + public void on_predict_begin() + { + _reset_progbar(); + _maybe_init_progbar(); + } + + public void on_predict_batch_begin(long step) + { + + } + + public void on_predict_batch_end(long end_step, Dictionary logs) + { + + } + + public void on_predict_end() + { + + } } } diff --git a/src/TensorFlowNET.Keras/Engine/Model.Predict.cs b/src/TensorFlowNET.Keras/Engine/Model.Predict.cs index 6dbce98cc..4d5755b0a 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Predict.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Predict.cs @@ -5,11 +5,70 @@ using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine.DataAdapters; using static Tensorflow.Binding; +using Tensorflow.Keras.Callbacks; namespace Tensorflow.Keras.Engine { public partial class Model { + public Tensors predict(IDatasetV2 dataset, + int batch_size = -1, + int verbose = 0, + int steps = -1, + int max_queue_size = 10, + int workers = 1, + bool use_multiprocessing = false) + { + var data_handler = new DataHandler(new DataHandlerArgs + { + Dataset = dataset, + BatchSize = batch_size, + StepsPerEpoch = steps, + InitialEpoch = 0, + Epochs = 1, + MaxQueueSize = max_queue_size, + Workers = workers, + UseMultiprocessing = use_multiprocessing, + Model = this, + StepsPerExecution = _steps_per_execution + }); + + var callbacks = new CallbackList(new CallbackParams + { + Model = this, + Verbose = verbose, + Epochs = 1, + Steps = data_handler.Inferredsteps + }); + + Tensor batch_outputs = null; + _predict_counter.assign(0); + callbacks.on_predict_begin(); + foreach (var (epoch, iterator) in data_handler.enumerate_epochs()) + { + foreach (var step in data_handler.steps()) + { + callbacks.on_predict_batch_begin(step); + var tmp_batch_outputs = run_predict_step(iterator); + if (batch_outputs == null) + { + batch_outputs = tmp_batch_outputs[0]; + } + else + { + batch_outputs = tf.concat(new Tensor[] { batch_outputs, tmp_batch_outputs[0] }, axis: 0); + } + + var end_step = step + data_handler.StepIncrement; + callbacks.on_predict_batch_end(end_step, new Dictionary { { "outputs", batch_outputs } }); + } + GC.Collect(); + } + + callbacks.on_predict_end(); + return batch_outputs; + } + /// /// Generates output predictions for the input samples. /// From 0ee50d319e5539f15b13f8909fd246c18819d840 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sun, 5 Feb 2023 20:38:52 -0600 Subject: [PATCH 22/52] Add double to NDArrayConverter. --- .../NumPy/NDArrayConverter.cs | 17 ++-- .../Engine/Model.Predict.cs | 80 ++++++++----------- src/TensorFlowNET.Keras/Engine/Model.cs | 1 - 3 files changed, 45 insertions(+), 53 deletions(-) diff --git a/src/TensorFlowNET.Core/NumPy/NDArrayConverter.cs b/src/TensorFlowNET.Core/NumPy/NDArrayConverter.cs index 2d042a5d1..c8c2d45fa 100644 --- a/src/TensorFlowNET.Core/NumPy/NDArrayConverter.cs +++ b/src/TensorFlowNET.Core/NumPy/NDArrayConverter.cs @@ -14,7 +14,8 @@ public unsafe static T Scalar(NDArray nd) where T : unmanaged TF_DataType.TF_FLOAT => Scalar(*(float*)nd.data), TF_DataType.TF_INT32 => Scalar(*(int*)nd.data), TF_DataType.TF_INT64 => Scalar(*(long*)nd.data), - _ => throw new NotImplementedException("") + TF_DataType.TF_DOUBLE => Scalar(*(double*)nd.data), + _ => throw new NotImplementedException(nameof(NDArrayConverter)) }; static T Scalar(byte input) @@ -23,7 +24,8 @@ static T Scalar(byte input) TypeCode.Byte => (T)Convert.ChangeType(input, TypeCode.Byte), TypeCode.Int32 => (T)Convert.ChangeType(input, TypeCode.Int32), TypeCode.Single => (T)Convert.ChangeType(input, TypeCode.Single), - _ => throw new NotImplementedException("") + TypeCode.Double => (T)Convert.ChangeType(input, TypeCode.Double), + _ => throw new NotImplementedException(nameof(NDArrayConverter)) }; static T Scalar(float input) @@ -32,7 +34,8 @@ static T Scalar(float input) TypeCode.Byte => (T)Convert.ChangeType(input, TypeCode.Byte), TypeCode.Int32 => (T)Convert.ChangeType(input, TypeCode.Int32), TypeCode.Single => (T)Convert.ChangeType(input, TypeCode.Single), - _ => throw new NotImplementedException("") + TypeCode.Double => (T)Convert.ChangeType(input, TypeCode.Double), + _ => throw new NotImplementedException(nameof(NDArrayConverter)) }; static T Scalar(int input) @@ -41,7 +44,8 @@ static T Scalar(int input) TypeCode.Byte => (T)Convert.ChangeType(input, TypeCode.Byte), TypeCode.Int64 => (T)Convert.ChangeType(input, TypeCode.Int64), TypeCode.Single => (T)Convert.ChangeType(input, TypeCode.Single), - _ => throw new NotImplementedException("") + TypeCode.Double => (T)Convert.ChangeType(input, TypeCode.Double), + _ => throw new NotImplementedException(nameof(NDArrayConverter)) }; static T Scalar(long input) @@ -50,7 +54,8 @@ static T Scalar(long input) TypeCode.Byte => (T)Convert.ChangeType(input, TypeCode.Byte), TypeCode.Int32 => (T)Convert.ChangeType(input, TypeCode.Int32), TypeCode.Single => (T)Convert.ChangeType(input, TypeCode.Single), - _ => throw new NotImplementedException("") + TypeCode.Double => (T)Convert.ChangeType(input, TypeCode.Double), + _ => throw new NotImplementedException(nameof(NDArrayConverter)) }; public static unsafe Array ToMultiDimArray(NDArray nd) where T : unmanaged @@ -65,7 +70,7 @@ public static unsafe Array ToMultiDimArray(NDArray nd) where T : unmanaged T[,,,] array => Addr(array), T[,,,,] array => Addr(array), T[,,,,,] array => Addr(array), - _ => throw new NotImplementedException("") + _ => throw new NotImplementedException(nameof(NDArrayConverter)) }; System.Buffer.MemoryCopy(nd.data.ToPointer(), addr, nd.bytesize, nd.bytesize); diff --git a/src/TensorFlowNET.Keras/Engine/Model.Predict.cs b/src/TensorFlowNET.Keras/Engine/Model.Predict.cs index 4d5755b0a..c27ea9090 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Predict.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Predict.cs @@ -1,5 +1,4 @@ -using Tensorflow.NumPy; -using System; +using System; using System.Collections.Generic; using System.Linq; using Tensorflow.Keras.ArgsDefinition; @@ -33,40 +32,7 @@ public Tensors predict(IDatasetV2 dataset, StepsPerExecution = _steps_per_execution }); - var callbacks = new CallbackList(new CallbackParams - { - Model = this, - Verbose = verbose, - Epochs = 1, - Steps = data_handler.Inferredsteps - }); - - Tensor batch_outputs = null; - _predict_counter.assign(0); - callbacks.on_predict_begin(); - foreach (var (epoch, iterator) in data_handler.enumerate_epochs()) - { - foreach (var step in data_handler.steps()) - { - callbacks.on_predict_batch_begin(step); - var tmp_batch_outputs = run_predict_step(iterator); - if (batch_outputs == null) - { - batch_outputs = tmp_batch_outputs[0]; - } - else - { - batch_outputs = tf.concat(new Tensor[] { batch_outputs, tmp_batch_outputs[0] }, axis: 0); - } - - var end_step = step + data_handler.StepIncrement; - callbacks.on_predict_batch_end(end_step, new Dictionary { { "outputs", batch_outputs } }); - } - GC.Collect(); - } - - callbacks.on_predict_end(); - return batch_outputs; + return PredictInternal(data_handler, verbose); } /// @@ -105,23 +71,45 @@ public Tensors predict(Tensor x, StepsPerExecution = _steps_per_execution }); - Tensors outputs = null; + return PredictInternal(data_handler, verbose); + } + + Tensors PredictInternal(DataHandler data_handler, int verbose) + { + var callbacks = new CallbackList(new CallbackParams + { + Model = this, + Verbose = verbose, + Epochs = 1, + Steps = data_handler.Inferredsteps + }); + + Tensor batch_outputs = null; _predict_counter.assign(0); - // callbacks.on_predict_begin() + callbacks.on_predict_begin(); foreach (var (epoch, iterator) in data_handler.enumerate_epochs()) { - foreach(var step in data_handler.steps()) + foreach (var step in data_handler.steps()) { - // callbacks.on_predict_batch_begin(step) - var batch_outputs = run_predict_step(iterator); - outputs = batch_outputs; + callbacks.on_predict_batch_begin(step); + var tmp_batch_outputs = run_predict_step(iterator); + if (batch_outputs == null) + { + batch_outputs = tmp_batch_outputs[0]; + } + else + { + batch_outputs = tf.concat(new Tensor[] { batch_outputs, tmp_batch_outputs[0] }, axis: 0); + } + var end_step = step + data_handler.StepIncrement; - // callbacks.on_predict_batch_end(end_step, {'outputs': batch_outputs}) + callbacks.on_predict_batch_end(end_step, new Dictionary { { "outputs", batch_outputs } }); } - GC.Collect(); } - // callbacks.on_predict_end() - return outputs; + + callbacks.on_predict_end(); + + return batch_outputs; } Tensors run_predict_step(OwnedIterator iterator) diff --git a/src/TensorFlowNET.Keras/Engine/Model.cs b/src/TensorFlowNET.Keras/Engine/Model.cs index dfe5b05f3..dd3e11a27 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.cs @@ -36,7 +36,6 @@ public partial class Model : Layer, IModel IVariableV1 _predict_counter; bool _base_model_initialized; bool stop_training; - DataHandler data_handler; public OptimizerV2 Optimizer { From a7c9a75954d219cb606042fcbfbeb1b176781d7e Mon Sep 17 00:00:00 2001 From: Superpiffer Date: Mon, 6 Feb 2023 12:42:56 +0100 Subject: [PATCH 23/52] Use a local Status variable Using a local reference ensure that the Status object cannot be disposed before the Dispose. This way it's also possible to use an external Status instance instead of the static one, if needed. --- .../Sessions/BaseSession.cs | 29 ++++++------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/src/TensorFlowNET.Core/Sessions/BaseSession.cs b/src/TensorFlowNET.Core/Sessions/BaseSession.cs index 095187b9a..4e131b365 100644 --- a/src/TensorFlowNET.Core/Sessions/BaseSession.cs +++ b/src/TensorFlowNET.Core/Sessions/BaseSession.cs @@ -30,6 +30,7 @@ namespace Tensorflow public class BaseSession : DisposableObject { protected Graph _graph; + protected Status _status; public Graph graph => _graph; public BaseSession(IntPtr handle, Graph g) @@ -48,9 +49,9 @@ public BaseSession(string target = "", Graph g = null, ConfigProto config = null } using var opts = new SessionOptions(target, config); - status = status ?? tf.Status; - _handle = c_api.TF_NewSession(_graph, opts.Handle, status.Handle); - status.Check(true); + _status = status ?? tf.Status; + _handle = c_api.TF_NewSession(_graph, opts.Handle, _status.Handle); + _status.Check(true); } public virtual void run(Operation op, params FeedItem[] feed_dict) @@ -217,8 +218,6 @@ private unsafe NDArray[] _call_tf_sessionrun(KeyValuePair[] f // Ensure any changes to the graph are reflected in the runtime. _extend_graph(); - var status = tf.Status; - var output_values = fetch_list.Select(x => IntPtr.Zero).ToArray(); c_api.TF_SessionRun(_handle, @@ -232,9 +231,9 @@ private unsafe NDArray[] _call_tf_sessionrun(KeyValuePair[] f target_opers: target_list.Select(f => (IntPtr)f).ToArray(), ntargets: target_list.Count, run_metadata: IntPtr.Zero, - status: status.Handle); + status: _status.Handle); - status.Check(true); + _status.Check(true); var result = new NDArray[fetch_list.Length]; @@ -246,8 +245,6 @@ private unsafe NDArray[] _call_tf_sessionrun(KeyValuePair[] f public unsafe Tensor eval(Tensor tensor) { - var status = tf.Status; - var output_values = new IntPtr[1]; var fetch_list = new[] { tensor._as_tf_output() }; @@ -262,9 +259,9 @@ public unsafe Tensor eval(Tensor tensor) target_opers: new IntPtr[0], ntargets: 0, run_metadata: IntPtr.Zero, - status: status.Handle); + status: _status.Handle); - status.Check(true); + _status.Check(true); return new Tensor(new SafeTensorHandle(output_values[0])); } @@ -291,15 +288,7 @@ private void _extend_graph() protected override void DisposeUnmanagedResources(IntPtr handle) { // c_api.TF_CloseSession(handle, tf.Status.Handle); - if (tf.Status == null || tf.Status.Handle.IsInvalid) - { - using var status = new Status(); - c_api.TF_DeleteSession(handle, status.Handle); - } - else - { - c_api.TF_DeleteSession(handle, tf.Status.Handle); - } + c_api.TF_DeleteSession(handle, _status.Handle); } } } From a44028ed23ca85d3df75dc529e18d2d3c7bb4a62 Mon Sep 17 00:00:00 2001 From: AsakusaRinne Date: Wed, 8 Feb 2023 13:09:49 +0800 Subject: [PATCH 24/52] Fix ConvTraspose2D gradient and register rsqrt gradient. --- .../Gradients/Tape.ComputeGradient.cs | 8 +++++--- src/TensorFlowNET.Core/Gradients/math_grad.cs | 14 ++++++++++++++ src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs | 14 ++++++++++++++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/TensorFlowNET.Core/Gradients/Tape.ComputeGradient.cs b/src/TensorFlowNET.Core/Gradients/Tape.ComputeGradient.cs index 0d0ecbe25..73c9e501e 100644 --- a/src/TensorFlowNET.Core/Gradients/Tape.ComputeGradient.cs +++ b/src/TensorFlowNET.Core/Gradients/Tape.ComputeGradient.cs @@ -90,7 +90,7 @@ public Tensor[] ComputeGradient(Tensor[] target_tensor_ids, in_gradients = trace.backward_function(out_gradients.ToArray(), unneeded_gradients.ToArray()); - if (in_gradients.Count() != trace.input_tensor_id.Count()) + if (in_gradients.Length != trace.input_tensor_id.Length && in_gradients.Length + unneeded_gradients.Count != trace.input_tensor_id.Length) throw new RuntimeError($"Recorded operation '{trace.op_type}' returned too few gradients. Expected {trace.input_tensor_id.Length} but received {in_gradients.Count()}"); if (!_persistent) { @@ -103,9 +103,11 @@ public Tensor[] ComputeGradient(Tensor[] target_tensor_ids, in_gradients = new Tensor[trace.input_tensor_id.Length]; } - for (int i = 0; i < in_gradients.Length; ++i) + bool skip_unneeded_id = trace.input_tensor_id.Length > in_gradients.Length; + for (int i = 0, k = 0; i < in_gradients.Length && k < trace.input_tensor_id.Count(); ++i, ++k) { - var id = trace.input_tensor_id[i]; + if (skip_unneeded_id && unneeded_gradients.Contains(k)) ++k; + var id = trace.input_tensor_id[k]; if (in_gradients[i] != null) { var unaggregated_grads = gradients[id]; diff --git a/src/TensorFlowNET.Core/Gradients/math_grad.cs b/src/TensorFlowNET.Core/Gradients/math_grad.cs index d9bc9b228..22d3c641b 100644 --- a/src/TensorFlowNET.Core/Gradients/math_grad.cs +++ b/src/TensorFlowNET.Core/Gradients/math_grad.cs @@ -639,6 +639,20 @@ public static Tensor[] _SqrtGrad(Operation op, Tensor[] grads) }); } + [RegisterGradient("Rsqrt")] + public static Tensor[] _RsqrtGrad(Operation op, Tensor[] grads) + { + var grad = grads[0]; + var y = op.outputs[0]; + + return tf_with(ops.control_dependencies(grads), delegate + { + y = math_ops.conj(y); + var factor = constant_op.constant(-0.5f, dtype: y.dtype); + return new Tensor[] { grad * (factor * math_ops.square(y) * y) }; + }); + } + [RegisterGradient("Asin")] public static Tensor[] _ASinGrad(Operation op, Tensor[] grads) { diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs index 525bfd354..f1860da1b 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs @@ -55,6 +55,20 @@ public ILayer Conv2D(int filters, IRegularizer bias_regularizer = null, IRegularizer activity_regularizer = null); + public ILayer Conv2DTranspose(int filters, + Shape kernel_size = null, + Shape strides = null, + string output_padding = "valid", + string data_format = null, + Shape dilation_rate = null, + string activation = null, + bool use_bias = true, + string kernel_initializer = null, + string bias_initializer = null, + string kernel_regularizer = null, + string bias_regularizer = null, + string activity_regularizer = null); + public ILayer Conv2D(int filters, Shape kernel_size = null, Shape strides = null, From 8a21ad2816f9f0573060371f217bdb45c251c409 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 9 Feb 2023 20:43:23 -0600 Subject: [PATCH 25/52] Fix math_ops.max and min. --- .../Operations/gen_math_ops.cs | 25 ++++++++++--------- .../Tensorflow.Binding.csproj | 6 ++--- .../Tensorflow.Keras.csproj | 8 +++--- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/src/TensorFlowNET.Core/Operations/gen_math_ops.cs b/src/TensorFlowNET.Core/Operations/gen_math_ops.cs index 894f9780d..564abbd0f 100644 --- a/src/TensorFlowNET.Core/Operations/gen_math_ops.cs +++ b/src/TensorFlowNET.Core/Operations/gen_math_ops.cs @@ -480,26 +480,27 @@ public static Tensor _any(Tx input, Ty axis, bool keep_dims = false, str return _op.outputs[0]; } - /// - /// Subroutine for Min or Max functions. See _min and _max - /// - private static Tensor MinOrMax(Tx input, Ty axis, string methodName, bool keep_dims = false, string name = null) - => tf.Context.ExecuteOp(methodName, name, new ExecuteOpArgs(input, axis) + public static Tensor _max(Tx input, Ty axis, bool keep_dims = false, string name = null) + => tf.Context.ExecuteOp("Max", name, new ExecuteOpArgs(input, axis) { GetGradientAttrs = (op) => new { T = op.get_attr("T"), - align_corners = op.get_attr("align_corners"), - half_pixel_centers = op.get_attr("half_pixel_centers") + keep_dims = op.get_attr("keep_dims"), + Tidx = op.get_attr("Tidx") } }.SetAttributes(new { keep_dims, reduction_indices = axis })); - public static Tensor _max(Tx input, Ty axis, bool keep_dims = false, string name = null) - => MinOrMax(input, axis, "Max", keep_dims: keep_dims, name: name); - public static Tensor _min(Tx input, Ty axis, bool keep_dims = false, string name = null) - => MinOrMax(input, axis, "Min", keep_dims: keep_dims, name: name); - + => tf.Context.ExecuteOp("Min", name, new ExecuteOpArgs(input, axis) + { + GetGradientAttrs = (op) => new + { + T = op.get_attr("T"), + keep_dims = op.get_attr("keep_dims"), + Tidx = op.get_attr("Tidx") + } + }.SetAttributes(new { keep_dims, reduction_indices = axis })); public static Tensor pow(Tx x, Ty y, string name = null) => tf.Context.ExecuteOp("Pow", name, new ExecuteOpArgs(x, y)); diff --git a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj index ede72a6ae..c2b53e761 100644 --- a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj +++ b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj @@ -5,7 +5,7 @@ Tensorflow.Binding Tensorflow 2.10.0 - 0.100.2 + 0.100.3 10.0 enable Haiping Chen, Meinrad Recheis, Eli Belash @@ -20,7 +20,7 @@ Google's TensorFlow full binding in .NET Standard. Building, training and infering deep learning models. https://tensorflownet.readthedocs.io - 0.100.2.0 + 0.100.3.0 tf.net 0.100.x and above are based on tensorflow native 2.10.0 @@ -38,7 +38,7 @@ https://tensorflownet.readthedocs.io tf.net 0.7x.x aligns with TensorFlow v2.7.x native library. tf.net 0.10x.x aligns with TensorFlow v2.10.x native library. - 0.100.2.0 + 0.100.3.0 LICENSE true true diff --git a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj index f7d186355..264b9501e 100644 --- a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj +++ b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj @@ -7,10 +7,10 @@ enable Tensorflow.Keras AnyCPU;x64 - 0.10.2 + 0.10.3 Haiping Chen Keras for .NET - Apache 2.0, Haiping Chen 2021 + Apache 2.0, Haiping Chen 2023 TensorFlow.Keras https://github.com/SciSharp/TensorFlow.NET https://avatars3.githubusercontent.com/u/44989469?s=200&v=4 @@ -37,8 +37,8 @@ Keras is an API designed for human beings, not machines. Keras follows best prac Git true Open.snk - 0.10.2.0 - 0.10.2.0 + 0.10.3.0 + 0.10.3.0 LICENSE Debug;Release;GPU From ccda2c39ff27c37c692ac5318711676ec339fbdd Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 16 Feb 2023 07:53:21 -0600 Subject: [PATCH 26/52] Add BinaryCrossentropy loss function. --- src/TensorFlowNET.Core/GlobalUsing.cs | 3 ++ src/TensorFlowNET.Core/Keras/IKerasApi.cs | 2 + .../Keras/Losses/ILossFunc.cs | 8 +++ .../Keras/Losses/ILossesApi.cs | 41 +++++++++++++++ src/TensorFlowNET.Keras/BackendImpl.cs | 14 ++++++ src/TensorFlowNET.Keras/GlobalUsing.cs | 5 ++ src/TensorFlowNET.Keras/KerasInterface.cs | 2 +- .../Losses/BinaryCrossentropy.cs | 24 +++++++++ .../Losses/CategoricalCrossentropy.cs | 41 +++++++-------- src/TensorFlowNET.Keras/Losses/ILossFunc.cs | 9 ---- src/TensorFlowNET.Keras/Losses/Loss.cs | 14 +++++- src/TensorFlowNET.Keras/Losses/LossesApi.cs | 17 +++++-- src/TensorFlowNET.Keras/Utils/losses_utils.cs | 28 ++++------- .../Losses/LossesTest.cs | 50 +++++++++++++++++++ .../Tensorflow.Keras.UnitTest.csproj | 2 +- 15 files changed, 202 insertions(+), 58 deletions(-) create mode 100644 src/TensorFlowNET.Core/GlobalUsing.cs create mode 100644 src/TensorFlowNET.Core/Keras/Losses/ILossFunc.cs create mode 100644 src/TensorFlowNET.Core/Keras/Losses/ILossesApi.cs create mode 100644 src/TensorFlowNET.Keras/GlobalUsing.cs create mode 100644 src/TensorFlowNET.Keras/Losses/BinaryCrossentropy.cs delete mode 100644 src/TensorFlowNET.Keras/Losses/ILossFunc.cs create mode 100644 test/TensorFlowNET.Keras.UnitTest/Losses/LossesTest.cs diff --git a/src/TensorFlowNET.Core/GlobalUsing.cs b/src/TensorFlowNET.Core/GlobalUsing.cs new file mode 100644 index 000000000..fe77202ce --- /dev/null +++ b/src/TensorFlowNET.Core/GlobalUsing.cs @@ -0,0 +1,3 @@ +global using System; +global using System.Collections.Generic; +global using System.Text; diff --git a/src/TensorFlowNET.Core/Keras/IKerasApi.cs b/src/TensorFlowNET.Core/Keras/IKerasApi.cs index 7f85f02f3..49ec9a5f2 100644 --- a/src/TensorFlowNET.Core/Keras/IKerasApi.cs +++ b/src/TensorFlowNET.Core/Keras/IKerasApi.cs @@ -2,12 +2,14 @@ using System.Collections.Generic; using System.Text; using Tensorflow.Keras.Layers; +using Tensorflow.Keras.Losses; namespace Tensorflow.Keras { public interface IKerasApi { public ILayersApi layers { get; } + public ILossesApi losses { get; } public IInitializersApi initializers { get; } } } diff --git a/src/TensorFlowNET.Core/Keras/Losses/ILossFunc.cs b/src/TensorFlowNET.Core/Keras/Losses/ILossFunc.cs new file mode 100644 index 000000000..408c7ca18 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Losses/ILossFunc.cs @@ -0,0 +1,8 @@ +namespace Tensorflow.Keras.Losses; + +public interface ILossFunc +{ + public string Reduction { get; } + public string Name { get; } + Tensor Call(Tensor y_true, Tensor y_pred, Tensor sample_weight = null); +} diff --git a/src/TensorFlowNET.Core/Keras/Losses/ILossesApi.cs b/src/TensorFlowNET.Core/Keras/Losses/ILossesApi.cs new file mode 100644 index 000000000..c42493368 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Losses/ILossesApi.cs @@ -0,0 +1,41 @@ +namespace Tensorflow.Keras.Losses; + +public interface ILossesApi +{ + ILossFunc BinaryCrossentropy(bool from_logits = false, + float label_smoothing = 0f, + int axis = -1, + string reduction = "auto", + string name = "binary_crossentropy"); + + ILossFunc SparseCategoricalCrossentropy(string reduction = null, + string name = null, + bool from_logits = false); + + ILossFunc CategoricalCrossentropy(string reduction = null, + string name = null, + bool from_logits = false); + + ILossFunc MeanSquaredError(string reduction = null, + string name = null); + + ILossFunc MeanSquaredLogarithmicError(string reduction = null, + string name = null); + + ILossFunc MeanAbsolutePercentageError(string reduction = null, + string name = null); + + ILossFunc MeanAbsoluteError(string reduction = null, + string name = null); + + ILossFunc CosineSimilarity(string reduction = null, + int axis = -1, + string name = null); + + ILossFunc Huber(string reduction = null, + string name = null, + Tensor delta = null); + + ILossFunc LogCosh(string reduction = null, + string name = null); +} diff --git a/src/TensorFlowNET.Keras/BackendImpl.cs b/src/TensorFlowNET.Keras/BackendImpl.cs index a62e8196e..0c9da015b 100644 --- a/src/TensorFlowNET.Keras/BackendImpl.cs +++ b/src/TensorFlowNET.Keras/BackendImpl.cs @@ -276,6 +276,20 @@ public Tensor categorical_crossentropy(Tensor target, Tensor output, bool from_l return -math_ops.reduce_sum(target * math_ops.log(output), new Axis(axis)); } + public Tensor binary_crossentropy(Tensor target, Tensor output, bool from_logits = false) + { + if (from_logits) + return tf.nn.sigmoid_cross_entropy_with_logits(labels: target, logits: output); + + var epsilon_ = constant_op.constant(epsilon(), dtype: output.dtype.as_base_dtype()); + output = tf.clip_by_value(output, epsilon_, 1.0f - epsilon_); + + // Compute cross entropy from probabilities. + var bce = target * tf.math.log(output + epsilon()); + bce += (1 - target) * tf.math.log(1 - output + epsilon()); + return -bce; + } + /// /// Resizes the images contained in a 4D tensor. /// diff --git a/src/TensorFlowNET.Keras/GlobalUsing.cs b/src/TensorFlowNET.Keras/GlobalUsing.cs new file mode 100644 index 000000000..72ff8b289 --- /dev/null +++ b/src/TensorFlowNET.Keras/GlobalUsing.cs @@ -0,0 +1,5 @@ +global using System; +global using System.Collections.Generic; +global using System.Text; +global using static Tensorflow.Binding; +global using static Tensorflow.KerasApi; \ No newline at end of file diff --git a/src/TensorFlowNET.Keras/KerasInterface.cs b/src/TensorFlowNET.Keras/KerasInterface.cs index 8dde1ab41..4e0c612bb 100644 --- a/src/TensorFlowNET.Keras/KerasInterface.cs +++ b/src/TensorFlowNET.Keras/KerasInterface.cs @@ -21,7 +21,7 @@ public class KerasInterface : IKerasApi public IInitializersApi initializers { get; } = new InitializersApi(); public Regularizers regularizers { get; } = new Regularizers(); public ILayersApi layers { get; } = new LayersApi(); - public LossesApi losses { get; } = new LossesApi(); + public ILossesApi losses { get; } = new LossesApi(); public Activations activations { get; } = new Activations(); public Preprocessing preprocessing { get; } = new Preprocessing(); ThreadLocal _backend = new ThreadLocal(() => new BackendImpl()); diff --git a/src/TensorFlowNET.Keras/Losses/BinaryCrossentropy.cs b/src/TensorFlowNET.Keras/Losses/BinaryCrossentropy.cs new file mode 100644 index 000000000..ff7bb6b70 --- /dev/null +++ b/src/TensorFlowNET.Keras/Losses/BinaryCrossentropy.cs @@ -0,0 +1,24 @@ +namespace Tensorflow.Keras.Losses; + +public class BinaryCrossentropy : LossFunctionWrapper, ILossFunc +{ + float label_smoothing; + public BinaryCrossentropy( + bool from_logits = false, + float label_smoothing = 0, + string reduction = null, + string name = null) : + base(reduction: reduction, + name: name == null ? "binary_crossentropy" : name, + from_logits: from_logits) + { + this.label_smoothing = label_smoothing; + } + + + public override Tensor Apply(Tensor y_true, Tensor y_pred, bool from_logits = false, int axis = -1) + { + var sum = keras.backend.binary_crossentropy(y_true, y_pred, from_logits: from_logits); + return keras.backend.mean(sum, axis: axis); + } +} diff --git a/src/TensorFlowNET.Keras/Losses/CategoricalCrossentropy.cs b/src/TensorFlowNET.Keras/Losses/CategoricalCrossentropy.cs index c80b1a83d..feb052244 100644 --- a/src/TensorFlowNET.Keras/Losses/CategoricalCrossentropy.cs +++ b/src/TensorFlowNET.Keras/Losses/CategoricalCrossentropy.cs @@ -1,31 +1,24 @@ -using System; -using System.Collections.Generic; -using System.Text; -using static Tensorflow.Binding; -using static Tensorflow.KerasApi; +namespace Tensorflow.Keras.Losses; -namespace Tensorflow.Keras.Losses +public class CategoricalCrossentropy : LossFunctionWrapper, ILossFunc { - public class CategoricalCrossentropy : LossFunctionWrapper, ILossFunc + float label_smoothing; + public CategoricalCrossentropy( + bool from_logits = false, + float label_smoothing = 0, + string reduction = null, + string name = null) : + base(reduction: reduction, + name: name == null ? "categorical_crossentropy" : name, + from_logits: from_logits) { - float label_smoothing; - public CategoricalCrossentropy( - bool from_logits = false, - float label_smoothing = 0, - string reduction = null, - string name = null) : - base(reduction: reduction, - name: name == null ? "categorical_crossentropy" : name, - from_logits: from_logits) - { - this.label_smoothing = label_smoothing; - } + this.label_smoothing = label_smoothing; + } - public override Tensor Apply(Tensor y_true, Tensor y_pred, bool from_logits = false, int axis = -1) - { - // Try to adjust the shape so that rank of labels = rank of logits - 1. - return keras.backend.categorical_crossentropy(y_true, y_pred, from_logits: from_logits); - } + public override Tensor Apply(Tensor y_true, Tensor y_pred, bool from_logits = false, int axis = -1) + { + // Try to adjust the shape so that rank of labels = rank of logits - 1. + return keras.backend.categorical_crossentropy(y_true, y_pred, from_logits: from_logits); } } diff --git a/src/TensorFlowNET.Keras/Losses/ILossFunc.cs b/src/TensorFlowNET.Keras/Losses/ILossFunc.cs deleted file mode 100644 index 8bc226df8..000000000 --- a/src/TensorFlowNET.Keras/Losses/ILossFunc.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Tensorflow.Keras.Losses -{ - public interface ILossFunc - { - public string Reduction { get; } - public string Name { get; } - Tensor Call(Tensor y_true, Tensor y_pred, Tensor sample_weight = null); - } -} diff --git a/src/TensorFlowNET.Keras/Losses/Loss.cs b/src/TensorFlowNET.Keras/Losses/Loss.cs index fe017ac42..77bf7e1dc 100644 --- a/src/TensorFlowNET.Keras/Losses/Loss.cs +++ b/src/TensorFlowNET.Keras/Losses/Loss.cs @@ -16,7 +16,7 @@ public abstract class Loss public string Reduction => reduction; public string Name => name; - public Loss(string reduction = ReductionV2.AUTO, + public Loss(string reduction = ReductionV2.AUTO, string name = null, bool from_logits = false) { @@ -34,7 +34,17 @@ public virtual Tensor Apply(Tensor y_true, Tensor y_pred, bool from_logits = fal public Tensor Call(Tensor y_true, Tensor y_pred, Tensor sample_weight = null) { var losses = Apply(y_true, y_pred, from_logits: from_logits); - return losses_utils.compute_weighted_loss(losses, reduction: this.reduction , sample_weight: sample_weight); + var reduction = GetReduction(); + return losses_utils.compute_weighted_loss(losses, reduction: reduction, sample_weight: sample_weight); + } + + string GetReduction() + { + return reduction switch + { + ReductionV2.AUTO => ReductionV2.SUM_OVER_BATCH_SIZE, + _ => reduction + }; } void _set_name_scope() diff --git a/src/TensorFlowNET.Keras/Losses/LossesApi.cs b/src/TensorFlowNET.Keras/Losses/LossesApi.cs index 71cffebb6..29e15e53c 100644 --- a/src/TensorFlowNET.Keras/Losses/LossesApi.cs +++ b/src/TensorFlowNET.Keras/Losses/LossesApi.cs @@ -1,7 +1,17 @@ namespace Tensorflow.Keras.Losses { - public class LossesApi + public class LossesApi : ILossesApi { + public ILossFunc BinaryCrossentropy(bool from_logits = false, + float label_smoothing = 0, + int axis = -1, + string reduction = "auto", + string name = "binary_crossentropy") + => new BinaryCrossentropy(from_logits: from_logits, + label_smoothing: label_smoothing, + reduction: reduction, + name: name); + public ILossFunc SparseCategoricalCrossentropy(string reduction = null, string name = null,bool from_logits = false) => new SparseCategoricalCrossentropy(reduction: reduction, name: name,from_logits: from_logits); @@ -19,14 +29,13 @@ public ILossFunc MeanAbsolutePercentageError(string reduction = null, string nam public ILossFunc MeanAbsoluteError(string reduction = null, string name = null) => new MeanAbsoluteError(reduction: reduction, name: name); - public ILossFunc CosineSimilarity(string reduction = null, string name = null,int axis=-1) - => new CosineSimilarity(reduction: reduction, name: name, axis: axis); + public ILossFunc CosineSimilarity(string reduction = null, int axis = -1, string name = null) + => new CosineSimilarity(reduction: reduction, axis: axis, name: name); public ILossFunc Huber(string reduction = null, string name = null, Tensor delta=null) => new Huber(reduction: reduction, name: name, delta: delta); public ILossFunc LogCosh(string reduction = null, string name = null) => new LogCosh(reduction: reduction, name: name); - } } diff --git a/src/TensorFlowNET.Keras/Utils/losses_utils.cs b/src/TensorFlowNET.Keras/Utils/losses_utils.cs index 8a8772fd0..083305954 100644 --- a/src/TensorFlowNET.Keras/Utils/losses_utils.cs +++ b/src/TensorFlowNET.Keras/Utils/losses_utils.cs @@ -24,23 +24,17 @@ public class losses_utils { public static Tensor compute_weighted_loss(Tensor losses, Tensor sample_weight = null, string reduction = null, string name = null) { - if (sample_weight == null) - sample_weight = losses.dtype == TF_DataType.TF_DOUBLE ? tf.constant(1.0) : tf.constant(1.0f); - var weighted_losses = scale_losses_by_sample_weight(losses, sample_weight); - // Apply reduction function to the individual weighted losses. - var loss = reduce_weighted_loss(weighted_losses, reduction); - // Convert the result back to the input type. - // loss = math_ops.cast(loss, losses.dtype); - return loss; - } - - public static Tensor scale_losses_by_sample_weight(Tensor losses, Tensor sample_weight) - { - // losses = math_ops.cast(losses, dtypes.float32); - // sample_weight = math_ops.cast(sample_weight, dtypes.float32); - // Update dimensions of `sample_weight` to match with `losses` if possible. - // (losses, sample_weight) = squeeze_or_expand_dimensions(losses, sample_weight); - return math_ops.multiply(losses, sample_weight); + return tf_with(ops.name_scope("weighted_loss"), scope => + { + if (sample_weight == null) + sample_weight = losses.dtype == TF_DataType.TF_DOUBLE ? tf.constant(1.0) : tf.constant(1.0f); + var weighted_losses = math_ops.multiply(losses, sample_weight); + // Apply reduction function to the individual weighted losses. + var loss = reduce_weighted_loss(weighted_losses, reduction); + // Convert the result back to the input type. + // loss = math_ops.cast(loss, losses.dtype); + return loss; + }); } public static (Tensor, Tensor) squeeze_or_expand_dimensions(Tensor y_pred, Tensor sample_weight) diff --git a/test/TensorFlowNET.Keras.UnitTest/Losses/LossesTest.cs b/test/TensorFlowNET.Keras.UnitTest/Losses/LossesTest.cs new file mode 100644 index 000000000..dad46c552 --- /dev/null +++ b/test/TensorFlowNET.Keras.UnitTest/Losses/LossesTest.cs @@ -0,0 +1,50 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using TensorFlowNET.Keras.UnitTest; +using static Tensorflow.Binding; +using static Tensorflow.KerasApi; + +namespace Tensorflow.Keras.UnitTest.Losses; + +[TestClass] +public class LossesTest : EagerModeTestBase +{ + /// + /// https://www.tensorflow.org/api_docs/python/tf/keras/losses/BinaryCrossentropy + /// + [TestMethod] + public void BinaryCrossentropy() + { + // Example 1: (batch_size = 1, number of samples = 4) + var y_true = tf.constant(new float[] { 0, 1, 0, 0 }); + var y_pred = tf.constant(new float[] { -18.6f, 0.51f, 2.94f, -12.8f }); + var bce = tf.keras.losses.BinaryCrossentropy(from_logits: true); + var loss = bce.Call(y_true, y_pred); + Assert.AreEqual((float)loss, 0.865458f); + + // Example 2: (batch_size = 2, number of samples = 4) + y_true = tf.constant(new float[,] { { 0, 1 }, { 0, 0 } }); + y_pred = tf.constant(new float[,] { { -18.6f, 0.51f }, { 2.94f, -12.8f } }); + bce = tf.keras.losses.BinaryCrossentropy(from_logits: true); + loss = bce.Call(y_true, y_pred); + Assert.AreEqual((float)loss, 0.865458f); + + // Using 'sample_weight' attribute + loss = bce.Call(y_true, y_pred, sample_weight: tf.constant(new[] { 0.8f, 0.2f })); + Assert.AreEqual((float)loss, 0.2436386f); + + // Using 'sum' reduction` type. + bce = tf.keras.losses.BinaryCrossentropy(from_logits: true, reduction: Reduction.SUM); + loss = bce.Call(y_true, y_pred); + Assert.AreEqual((float)loss, 1.730916f); + + // Using 'none' reduction type. + bce = tf.keras.losses.BinaryCrossentropy(from_logits: true, reduction: Reduction.NONE); + loss = bce.Call(y_true, y_pred); + Assert.AreEqual(new float[] { 0.23515666f, 1.4957594f}, loss.numpy()); + } +} diff --git a/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj b/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj index 61e522e6c..c9020f7b4 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj +++ b/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj @@ -1,4 +1,4 @@ - + net6.0 From 5821275145e5123d1acbc4094acd2baef06ac138 Mon Sep 17 00:00:00 2001 From: Superpiffer Date: Fri, 10 Feb 2023 16:21:26 +0100 Subject: [PATCH 27/52] Reimplemented NDArray == and != operators, handling null values. Added unit tests. --- .../NumPy/NDArray.Operators.cs | 22 ++++++++++--- .../NumPy/OperatorsTest.cs | 33 +++++++++++++++++++ 2 files changed, 51 insertions(+), 4 deletions(-) create mode 100644 test/TensorFlowNET.UnitTest/NumPy/OperatorsTest.cs diff --git a/src/TensorFlowNET.Core/NumPy/NDArray.Operators.cs b/src/TensorFlowNET.Core/NumPy/NDArray.Operators.cs index 7168678a3..ef3b76f73 100644 --- a/src/TensorFlowNET.Core/NumPy/NDArray.Operators.cs +++ b/src/TensorFlowNET.Core/NumPy/NDArray.Operators.cs @@ -25,10 +25,24 @@ public partial class NDArray [AutoNumPy] public static NDArray operator -(NDArray lhs) => new NDArray(gen_math_ops.neg(lhs)); [AutoNumPy] - public static NDArray operator ==(NDArray lhs, NDArray rhs) - => rhs is null ? Scalar(false) : new NDArray(math_ops.equal(lhs, rhs)); + public static NDArray operator ==(NDArray lhs, NDArray rhs) + { + if(ReferenceEquals(lhs, rhs)) + return Scalar(true); + if(lhs is null) + return Scalar(false); + if(rhs is null) + return Scalar(false); + return new NDArray(math_ops.equal(lhs, rhs)); + } [AutoNumPy] - public static NDArray operator !=(NDArray lhs, NDArray rhs) - => new NDArray(math_ops.not_equal(lhs, rhs)); + public static NDArray operator !=(NDArray lhs, NDArray rhs) + { + if(ReferenceEquals(lhs, rhs)) + return Scalar(false); + if(lhs is null || rhs is null) + return Scalar(true); + return new NDArray(math_ops.not_equal(lhs, rhs)); + } } } diff --git a/test/TensorFlowNET.UnitTest/NumPy/OperatorsTest.cs b/test/TensorFlowNET.UnitTest/NumPy/OperatorsTest.cs new file mode 100644 index 000000000..e4989a1dc --- /dev/null +++ b/test/TensorFlowNET.UnitTest/NumPy/OperatorsTest.cs @@ -0,0 +1,33 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Tensorflow.NumPy; + +namespace TensorFlowNET.UnitTest.NumPy +{ + [TestClass] + public class OperatorsTest + { + [TestMethod] + public void EqualToOperator() + { + NDArray n1 = null; + NDArray n2 = new NDArray(1); + + Assert.IsTrue(n1 == null); + Assert.IsFalse(n2 == null); + Assert.IsFalse(n1 == 1); + Assert.IsTrue(n2 == 1); + } + + [TestMethod] + public void NotEqualToOperator() + { + NDArray n1 = null; + NDArray n2 = new NDArray(1); + + Assert.IsFalse(n1 != null); + Assert.IsTrue(n2 != null); + Assert.IsTrue(n1 != 1); + Assert.IsFalse(n2 != 1); + } + } +} From 2ee08e8ce9a2e218260fbbe4925dd5ec9aa04f8e Mon Sep 17 00:00:00 2001 From: AsakusaRinne Date: Sat, 18 Feb 2023 13:15:37 +0800 Subject: [PATCH 28/52] Fix the keras.sparse_categorical_crossentropy. (#985) --- .../Losses/SparseCategoricalCrossentropy.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/TensorFlowNET.Keras/Losses/SparseCategoricalCrossentropy.cs b/src/TensorFlowNET.Keras/Losses/SparseCategoricalCrossentropy.cs index 0f6e4645b..b72412265 100644 --- a/src/TensorFlowNET.Keras/Losses/SparseCategoricalCrossentropy.cs +++ b/src/TensorFlowNET.Keras/Losses/SparseCategoricalCrossentropy.cs @@ -14,6 +14,13 @@ public override Tensor Apply(Tensor target, Tensor output, bool from_logits = fa { target = tf.cast(target, dtype: TF_DataType.TF_INT64); + if (!from_logits) + { + var epsilon = tf.constant(KerasApi.keras.backend.epsilon(), output.dtype); + output = tf.clip_by_value(output, epsilon, 1 - epsilon); + output = tf.log(output); + } + // Try to adjust the shape so that rank of labels = rank of logits - 1. var output_shape = array_ops.shape_v2(output); var output_rank = output.shape.ndim; From b8645d3f83f07692847b54af714c7c429f450a98 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sat, 18 Feb 2023 14:54:33 -0600 Subject: [PATCH 29/52] Add keras.layers.CategoryEncoding. --- .../Preprocessing/CategoryEncodingArgs.cs | 16 ++++ .../Keras/Layers/ILayersApi.cs | 12 +++ src/TensorFlowNET.Core/Operations/math_ops.cs | 16 +++- src/TensorFlowNET.Core/Tensors/constant_op.cs | 4 + src/TensorFlowNET.Keras/Layers/LayersApi.cs | 10 +++ .../Layers/Preprocessing/CategoryEncoding.cs | 75 +++++++++++++++++++ .../Layers/LayersTest.cs | 55 ++++++++++++++ .../Losses/LossesTest.cs | 3 +- 8 files changed, 189 insertions(+), 2 deletions(-) create mode 100644 src/TensorFlowNET.Core/Keras/ArgsDefinition/Preprocessing/CategoryEncodingArgs.cs create mode 100644 src/TensorFlowNET.Keras/Layers/Preprocessing/CategoryEncoding.cs diff --git a/src/TensorFlowNET.Core/Keras/ArgsDefinition/Preprocessing/CategoryEncodingArgs.cs b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Preprocessing/CategoryEncodingArgs.cs new file mode 100644 index 000000000..c282afd89 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/ArgsDefinition/Preprocessing/CategoryEncodingArgs.cs @@ -0,0 +1,16 @@ +using Newtonsoft.Json; +using Tensorflow.NumPy; + +namespace Tensorflow.Keras.ArgsDefinition +{ + public class CategoryEncodingArgs : AutoSerializeLayerArgs + { + [JsonProperty("num_tokens")] + public int NumTokens { get; set; } + [JsonProperty("output_mode")] + public string OutputMode { get; set; } + [JsonProperty("sparse")] + public bool Sparse { get; set; } + public NDArray CountWeights { get; set; } + } +} diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs index f1860da1b..9fcd0d70f 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs @@ -1,4 +1,5 @@ using System; +using Tensorflow.NumPy; using static Google.Protobuf.Reflection.FieldDescriptorProto.Types; namespace Tensorflow.Keras.Layers @@ -28,6 +29,17 @@ public ILayer BatchNormalization(int axis = -1, bool renorm = false, float renorm_momentum = 0.99f); + /// + /// A preprocessing layer which encodes integer features. + /// + /// The total number of tokens the layer should support. + /// Specification for the output of the layer. + /// + public ILayer CategoryEncoding(int num_tokens, + string output_mode = "one_hot", + bool sparse = false, + NDArray count_weights = null); + public ILayer Conv1D(int filters, Shape kernel_size, int strides = 1, diff --git a/src/TensorFlowNET.Core/Operations/math_ops.cs b/src/TensorFlowNET.Core/Operations/math_ops.cs index 861dba18b..9542f6436 100644 --- a/src/TensorFlowNET.Core/Operations/math_ops.cs +++ b/src/TensorFlowNET.Core/Operations/math_ops.cs @@ -839,10 +839,24 @@ public static Tensor bincount(Tensor arr, Tensor weights = null, output_size = math_ops.maximum(minlength, output_size); if (maxlength != null) output_size = math_ops.minimum(maxlength, output_size); - var weights = constant_op.constant(new long[0], dtype: dtype); + weights = weights ?? constant_op.constant(new int[0], dtype: dtype); return tf.Context.ExecuteOp("Bincount", name, new ExecuteOpArgs(arr, output_size, weights)); } + else + { + var array_is_nonempty = math_ops.reduce_prod(array_ops.shape(arr)) > 0; + var output_size = math_ops.cast(array_is_nonempty, arr.dtype) * (math_ops.reduce_max(arr) + 1); + if (minlength != null) + output_size = math_ops.maximum(minlength, output_size); + if (maxlength != null) + output_size = math_ops.minimum(maxlength, output_size); + weights = weights ?? array_ops.constant(new int[0], dtype: dtype); + return tf.Context.ExecuteOp("DenseBincount", name, + new ExecuteOpArgs(arr, output_size, weights, binary_output) + .SetAttributes(new { binary_output })); + } + throw new NotImplementedException(""); }); diff --git a/src/TensorFlowNET.Core/Tensors/constant_op.cs b/src/TensorFlowNET.Core/Tensors/constant_op.cs index 2c9035177..1a825e0cb 100644 --- a/src/TensorFlowNET.Core/Tensors/constant_op.cs +++ b/src/TensorFlowNET.Core/Tensors/constant_op.cs @@ -153,6 +153,10 @@ static Tensor convert_to_eager_tensor(object value, bool allow_broadcast) { var t = convert_to_eager_tensor(value, tf.Context, dtype: dtype); + if (dtype != TF_DataType.DtInvalid && dtype != t.dtype) + { + t = math_ops.cast(t, dtype); + } if (shape is null || shape.IsNull) return t; diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index 76634918d..0d71b2713 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -4,6 +4,7 @@ using Tensorflow.Keras.ArgsDefinition.Rnn; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Layers.Rnn; +using Tensorflow.NumPy; using static Tensorflow.Binding; using static Tensorflow.KerasApi; @@ -829,5 +830,14 @@ IInitializer GetInitializerByName(string name) "orthogonal" => tf.orthogonal_initializer, _ => tf.glorot_uniform_initializer }; + + public ILayer CategoryEncoding(int num_tokens, string output_mode = "one_hot", bool sparse = false, NDArray count_weights = null) + => new CategoryEncoding(new CategoryEncodingArgs + { + NumTokens = num_tokens, + OutputMode = output_mode, + Sparse = sparse, + CountWeights = count_weights + }); } } diff --git a/src/TensorFlowNET.Keras/Layers/Preprocessing/CategoryEncoding.cs b/src/TensorFlowNET.Keras/Layers/Preprocessing/CategoryEncoding.cs new file mode 100644 index 000000000..5620a916c --- /dev/null +++ b/src/TensorFlowNET.Keras/Layers/Preprocessing/CategoryEncoding.cs @@ -0,0 +1,75 @@ +using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.Engine; + +namespace Tensorflow.Keras.Layers +{ + /// + /// This layer provides options for condensing data into a categorical encoding when the total number of tokens are known in advance. + /// + public class CategoryEncoding : Layer + { + CategoryEncodingArgs args; + + public CategoryEncoding(CategoryEncodingArgs args) : base(args) + { + this.args = args; + } + + protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) + { + var depth = args.NumTokens; + var max_value = tf.reduce_max(inputs); + var min_value = tf.reduce_min(inputs); + + /*var condition = tf.logical_and(tf.greater(tf.cast(constant_op.constant(depth), max_value.dtype), max_value), + tf.greater_equal(min_value, tf.cast(constant_op.constant(0), min_value.dtype)));*/ + + var bincounts = encode_categorical_inputs(inputs, args.OutputMode, depth, args.DType, + sparse: args.Sparse, + count_weights: args.CountWeights); + + if(args.OutputMode != "tf_idf") + { + return bincounts; + } + + return inputs; + } + + public override Shape ComputeOutputShape(Shape input_shape) + { + return input_shape; + } + + Tensors encode_categorical_inputs(Tensor inputs, string output_mode, int depth, + TF_DataType dtype = TF_DataType.TF_FLOAT, + bool sparse = false, + Tensor count_weights = null) + { + bool binary_output = false; + if (output_mode == "one_hot") + { + binary_output = true; + if (inputs.shape[-1] != 1) + { + inputs = tf.expand_dims(inputs, -1); + } + } + else if (output_mode == "multi_hot") + { + binary_output = true; + } + + var depth_tensor = constant_op.constant(depth); + var result = tf.math.bincount(inputs, + weights: count_weights, + minlength: depth_tensor, + maxlength: depth_tensor, + dtype: dtype, + axis: -1, + binary_output: binary_output); + + return result; + } + } +} diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs index 029592c3f..f8a6174d9 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/LayersTest.cs @@ -177,5 +177,60 @@ public void LayerNormalization() Assert.AreEqual((5, 2), output.shape); Assert.IsTrue(output[0].numpy().Equals(new[] { -0.99998f, 0.99998f })); } + + /// + /// https://www.tensorflow.org/api_docs/python/tf/keras/layers/CategoryEncoding + /// + [TestMethod] + public void CategoryEncoding() + { + // one-hot + var inputs = np.array(new[] { 3, 2, 0, 1 }); + var layer = tf.keras.layers.CategoryEncoding(4); + + Tensor output = layer.Apply(inputs); + Assert.AreEqual((4, 4), output.shape); + Assert.IsTrue(output[0].numpy().Equals(new[] { 0, 0, 0, 1f })); + Assert.IsTrue(output[1].numpy().Equals(new[] { 0, 0, 1, 0f })); + Assert.IsTrue(output[2].numpy().Equals(new[] { 1, 0, 0, 0f })); + Assert.IsTrue(output[3].numpy().Equals(new[] { 0, 1, 0, 0f })); + + // multi-hot + inputs = np.array(new[,] + { + { 0, 1 }, + { 0, 0 }, + { 1, 2 }, + { 3, 1 } + }); + layer = tf.keras.layers.CategoryEncoding(4, output_mode: "multi_hot"); + output = layer.Apply(inputs); + Assert.IsTrue(output[0].numpy().Equals(new[] { 1, 1, 0, 0f })); + Assert.IsTrue(output[1].numpy().Equals(new[] { 1, 0, 0, 0f })); + Assert.IsTrue(output[2].numpy().Equals(new[] { 0, 1, 1, 0f })); + Assert.IsTrue(output[3].numpy().Equals(new[] { 0, 1, 0, 1f })); + + // using weighted inputs in "count" mode + inputs = np.array(new[,] + { + { 0, 1 }, + { 0, 0 }, + { 1, 2 }, + { 3, 1 } + }); + var weights = np.array(new[,] + { + { 0.1f, 0.2f }, + { 0.1f, 0.1f }, + { 0.2f, 0.3f }, + { 0.4f, 0.2f } + }); + layer = tf.keras.layers.CategoryEncoding(4, output_mode: "count", count_weights: weights); + output = layer.Apply(inputs); + Assert.IsTrue(output[0].numpy().Equals(new[] { 0.1f, 0.2f, 0f, 0f })); + Assert.IsTrue(output[1].numpy().Equals(new[] { 0.2f, 0f, 0f, 0f })); + Assert.IsTrue(output[2].numpy().Equals(new[] { 0f, 0.2f, 0.3f, 0f })); + Assert.IsTrue(output[3].numpy().Equals(new[] { 0f, 0.2f, 0f, 0.4f })); + } } } diff --git a/test/TensorFlowNET.Keras.UnitTest/Losses/LossesTest.cs b/test/TensorFlowNET.Keras.UnitTest/Losses/LossesTest.cs index dad46c552..b19f0203a 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Losses/LossesTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Losses/LossesTest.cs @@ -4,11 +4,12 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using Tensorflow; using TensorFlowNET.Keras.UnitTest; using static Tensorflow.Binding; using static Tensorflow.KerasApi; -namespace Tensorflow.Keras.UnitTest.Losses; +namespace TensorFlowNET.Keras.UnitTest; [TestClass] public class LossesTest : EagerModeTestBase From ca9f574fce755dd92f365d732b1ff1a20b568ecf Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sun, 19 Feb 2023 13:32:21 -0600 Subject: [PATCH 30/52] Add metric of top_k_categorical_accuracy. --- src/TensorFlowNET.Core/APIs/tf.math.cs | 3 ++ src/TensorFlowNET.Core/Keras/IKerasApi.cs | 2 + .../Keras/Metrics/IMetricsApi.cs | 29 ++++++++++++++ .../Operations/NnOps/gen_nn_ops.cs | 12 +----- src/TensorFlowNET.Core/Tensors/tensor_util.cs | 5 +++ src/TensorFlowNET.Keras/KerasInterface.cs | 2 +- src/TensorFlowNET.Keras/Metrics/MetricsApi.cs | 9 ++++- .../Metrics/metrics_utils.cs | 39 +++++++++++++++++++ .../Metrics/MetricsTest.cs | 28 +++++++++++++ 9 files changed, 117 insertions(+), 12 deletions(-) create mode 100644 src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs create mode 100644 src/TensorFlowNET.Keras/Metrics/metrics_utils.cs create mode 100644 test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs diff --git a/src/TensorFlowNET.Core/APIs/tf.math.cs b/src/TensorFlowNET.Core/APIs/tf.math.cs index ce6dc4d6c..7d3f6eff9 100644 --- a/src/TensorFlowNET.Core/APIs/tf.math.cs +++ b/src/TensorFlowNET.Core/APIs/tf.math.cs @@ -39,6 +39,9 @@ public Tensor erf(Tensor x, string name = null) public Tensor sum(Tensor x, Axis? axis = null, string name = null) => math_ops.reduce_sum(x, axis: axis, name: name); + public Tensor in_top_k(Tensor predictions, Tensor targets, int k, string name = "InTopK") + => nn_ops.in_top_k(predictions, targets, k, name); + /// /// /// diff --git a/src/TensorFlowNET.Core/Keras/IKerasApi.cs b/src/TensorFlowNET.Core/Keras/IKerasApi.cs index 49ec9a5f2..cffd3f797 100644 --- a/src/TensorFlowNET.Core/Keras/IKerasApi.cs +++ b/src/TensorFlowNET.Core/Keras/IKerasApi.cs @@ -3,6 +3,7 @@ using System.Text; using Tensorflow.Keras.Layers; using Tensorflow.Keras.Losses; +using Tensorflow.Keras.Metrics; namespace Tensorflow.Keras { @@ -10,6 +11,7 @@ public interface IKerasApi { public ILayersApi layers { get; } public ILossesApi losses { get; } + public IMetricsApi metrics { get; } public IInitializersApi initializers { get; } } } diff --git a/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs b/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs new file mode 100644 index 000000000..2fe6d8095 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs @@ -0,0 +1,29 @@ +namespace Tensorflow.Keras.Metrics; + +public interface IMetricsApi +{ + Tensor binary_accuracy(Tensor y_true, Tensor y_pred); + + Tensor categorical_accuracy(Tensor y_true, Tensor y_pred); + + Tensor mean_absolute_error(Tensor y_true, Tensor y_pred); + + Tensor mean_absolute_percentage_error(Tensor y_true, Tensor y_pred); + + /// + /// Calculates how often predictions matches integer labels. + /// + /// Integer ground truth values. + /// The prediction values. + /// Sparse categorical accuracy values. + Tensor sparse_categorical_accuracy(Tensor y_true, Tensor y_pred); + + /// + /// Computes how often targets are in the top `K` predictions. + /// + /// + /// + /// + /// + Tensor top_k_categorical_accuracy(Tensor y_true, Tensor y_pred, int k = 5); +} diff --git a/src/TensorFlowNET.Core/Operations/NnOps/gen_nn_ops.cs b/src/TensorFlowNET.Core/Operations/NnOps/gen_nn_ops.cs index 0567858f2..408d06ebf 100644 --- a/src/TensorFlowNET.Core/Operations/NnOps/gen_nn_ops.cs +++ b/src/TensorFlowNET.Core/Operations/NnOps/gen_nn_ops.cs @@ -240,16 +240,8 @@ public static Tensor log_softmax(Tensor logits, string name = null) /// /// A `Tensor` of type `bool`. public static Tensor in_top_kv2(Tensor predictions, Tensor targets, int k, string name = null) - { - var _op = tf.OpDefLib._apply_op_helper("InTopKV2", name: name, args: new - { - predictions, - targets, - k - }); - - return _op.output; - } + => tf.Context.ExecuteOp("InTopKV2", name, + new ExecuteOpArgs(predictions, targets, k)); public static Tensor leaky_relu(Tensor features, float alpha = 0.2f, string name = null) => tf.Context.ExecuteOp("LeakyRelu", name, diff --git a/src/TensorFlowNET.Core/Tensors/tensor_util.cs b/src/TensorFlowNET.Core/Tensors/tensor_util.cs index 5f09f202f..7af89f137 100644 --- a/src/TensorFlowNET.Core/Tensors/tensor_util.cs +++ b/src/TensorFlowNET.Core/Tensors/tensor_util.cs @@ -121,6 +121,11 @@ public static TensorProto make_tensor_proto(object values, TF_DataType dtype = T if (dtype == TF_DataType.TF_INT32) values = long_values.Select(x => (int)Convert.ChangeType(x, new_system_dtype)).ToArray(); } + else if (values is double[] double_values) + { + if (dtype == TF_DataType.TF_FLOAT) + values = double_values.Select(x => (float)Convert.ChangeType(x, new_system_dtype)).ToArray(); + } else values = Convert.ChangeType(values, new_system_dtype); diff --git a/src/TensorFlowNET.Keras/KerasInterface.cs b/src/TensorFlowNET.Keras/KerasInterface.cs index 4e0c612bb..e0d148cef 100644 --- a/src/TensorFlowNET.Keras/KerasInterface.cs +++ b/src/TensorFlowNET.Keras/KerasInterface.cs @@ -27,7 +27,7 @@ public class KerasInterface : IKerasApi ThreadLocal _backend = new ThreadLocal(() => new BackendImpl()); public BackendImpl backend => _backend.Value; public OptimizerApi optimizers { get; } = new OptimizerApi(); - public MetricsApi metrics { get; } = new MetricsApi(); + public IMetricsApi metrics { get; } = new MetricsApi(); public ModelsApi models { get; } = new ModelsApi(); public KerasUtils utils { get; } = new KerasUtils(); diff --git a/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs b/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs index 3d614e023..6b0e2d8a0 100644 --- a/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs +++ b/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs @@ -2,7 +2,7 @@ namespace Tensorflow.Keras.Metrics { - public class MetricsApi + public class MetricsApi : IMetricsApi { public Tensor binary_accuracy(Tensor y_true, Tensor y_pred) { @@ -53,5 +53,12 @@ public Tensor mean_absolute_percentage_error(Tensor y_true, Tensor y_pred) var diff = (y_true - y_pred) / math_ops.maximum(math_ops.abs(y_true), keras.backend.epsilon()); return 100f * keras.backend.mean(math_ops.abs(diff), axis: -1); } + + public Tensor top_k_categorical_accuracy(Tensor y_true, Tensor y_pred, int k = 5) + { + return metrics_utils.sparse_top_k_categorical_matches( + tf.math.argmax(y_true, axis: -1), y_pred, k + ); + } } } diff --git a/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs b/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs new file mode 100644 index 000000000..de6a8402e --- /dev/null +++ b/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs @@ -0,0 +1,39 @@ +using Tensorflow.NumPy; + +namespace Tensorflow.Keras.Metrics; + +public class metrics_utils +{ + public static Tensor sparse_top_k_categorical_matches(Tensor y_true, Tensor y_pred, int k = 5) + { + var reshape_matches = false; + var y_true_rank = y_true.shape.ndim; + var y_pred_rank = y_pred.shape.ndim; + var y_true_org_shape = tf.shape(y_true); + + if (y_pred_rank > 2) + { + y_pred = tf.reshape(y_pred, (-1, y_pred.shape[-1])); + } + + if (y_true_rank > 1) + { + reshape_matches = true; + y_true = tf.reshape(y_true, new Shape(-1)); + } + + var matches = tf.cast( + tf.math.in_top_k( + predictions: y_pred, targets: tf.cast(y_true, np.int32), k: k + ), + dtype: keras.backend.floatx() + ); + + if (reshape_matches) + { + return tf.reshape(matches, shape: y_true_org_shape); + } + + return matches; + } +} diff --git a/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs b/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs new file mode 100644 index 000000000..bb0107d4e --- /dev/null +++ b/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs @@ -0,0 +1,28 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Tensorflow; +using Tensorflow.NumPy; +using static Tensorflow.Binding; +using static Tensorflow.KerasApi; + +namespace TensorFlowNET.Keras.UnitTest; + +[TestClass] +public class MetricsTest : EagerModeTestBase +{ + /// + /// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/top_k_categorical_accuracy + /// + [TestMethod] + public void top_k_categorical_accuracy() + { + var y_true = np.array(new[,] { { 0, 0, 1 }, { 0, 1, 0 } }); + var y_pred = np.array(new[,] { { 0.1f, 0.9f, 0.8f }, { 0.05f, 0.95f, 0f } }); + var m = tf.keras.metrics.top_k_categorical_accuracy(y_true, y_pred, k: 3); + Assert.AreEqual(m.numpy(), new[] { 1f, 1f }); + } +} From a5289b9bb3ab98f54186d0627b2a8dde5c1e215e Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sun, 19 Feb 2023 15:43:47 -0600 Subject: [PATCH 31/52] Abstract IMetricFunc. --- .../Keras/Metrics/IMetricFunc.cs | 17 +++++++ .../Keras/Metrics/IMetricsApi.cs | 9 ++++ .../Metrics/MeanMetricWrapper.cs | 3 ++ src/TensorFlowNET.Keras/Metrics/Metric.cs | 2 +- src/TensorFlowNET.Keras/Metrics/MetricsApi.cs | 7 +-- .../Metrics/TopKCategoricalAccuracy.cs | 12 +++++ src/TensorFlowNET.Keras/Utils/losses_utils.cs | 45 ++++++++++++++++++- .../Metrics/MetricsTest.cs | 20 +++++++++ 8 files changed, 110 insertions(+), 5 deletions(-) create mode 100644 src/TensorFlowNET.Core/Keras/Metrics/IMetricFunc.cs create mode 100644 src/TensorFlowNET.Keras/Metrics/TopKCategoricalAccuracy.cs diff --git a/src/TensorFlowNET.Core/Keras/Metrics/IMetricFunc.cs b/src/TensorFlowNET.Core/Keras/Metrics/IMetricFunc.cs new file mode 100644 index 000000000..1867d6375 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Metrics/IMetricFunc.cs @@ -0,0 +1,17 @@ +namespace Tensorflow.Keras.Metrics; + +public interface IMetricFunc +{ + /// + /// Accumulates metric statistics. + /// + /// + /// + /// + /// + Tensor update_state(Tensor y_true, Tensor y_pred, Tensor sample_weight = null); + + Tensor result(); + + void reset_states(); +} diff --git a/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs b/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs index 2fe6d8095..511b0ef1b 100644 --- a/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs +++ b/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs @@ -26,4 +26,13 @@ public interface IMetricsApi /// /// Tensor top_k_categorical_accuracy(Tensor y_true, Tensor y_pred, int k = 5); + + /// + /// Computes how often targets are in the top K predictions. + /// + /// + /// + /// + /// + IMetricFunc TopKCategoricalAccuracy(int k = 5, string name = "top_k_categorical_accuracy", TF_DataType dtype = TF_DataType.TF_FLOAT); } diff --git a/src/TensorFlowNET.Keras/Metrics/MeanMetricWrapper.cs b/src/TensorFlowNET.Keras/Metrics/MeanMetricWrapper.cs index c422bfa64..2e985b88c 100644 --- a/src/TensorFlowNET.Keras/Metrics/MeanMetricWrapper.cs +++ b/src/TensorFlowNET.Keras/Metrics/MeanMetricWrapper.cs @@ -1,4 +1,5 @@ using System; +using Tensorflow.Keras.Utils; namespace Tensorflow.Keras.Metrics { @@ -17,6 +18,8 @@ public override Tensor update_state(Tensor y_true, Tensor y_pred, Tensor sample_ y_true = math_ops.cast(y_true, _dtype); y_pred = math_ops.cast(y_pred, _dtype); + (y_pred, y_true) = losses_utils.squeeze_or_expand_dimensions(y_pred, y_true: y_true); + var matches = _fn(y_true, y_pred); return update_state(matches, sample_weight: sample_weight); } diff --git a/src/TensorFlowNET.Keras/Metrics/Metric.cs b/src/TensorFlowNET.Keras/Metrics/Metric.cs index 21457f155..1dfc39c49 100644 --- a/src/TensorFlowNET.Keras/Metrics/Metric.cs +++ b/src/TensorFlowNET.Keras/Metrics/Metric.cs @@ -9,7 +9,7 @@ namespace Tensorflow.Keras.Metrics /// /// Encapsulates metric logic and state. /// - public class Metric : Layer + public class Metric : Layer, IMetricFunc { protected IVariableV1 total; protected IVariableV1 count; diff --git a/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs b/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs index 6b0e2d8a0..dfccfdbbe 100644 --- a/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs +++ b/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs @@ -1,6 +1,4 @@ -using static Tensorflow.KerasApi; - -namespace Tensorflow.Keras.Metrics +namespace Tensorflow.Keras.Metrics { public class MetricsApi : IMetricsApi { @@ -60,5 +58,8 @@ public Tensor top_k_categorical_accuracy(Tensor y_true, Tensor y_pred, int k = 5 tf.math.argmax(y_true, axis: -1), y_pred, k ); } + + public IMetricFunc TopKCategoricalAccuracy(int k = 5, string name = "top_k_categorical_accuracy", TF_DataType dtype = TF_DataType.TF_FLOAT) + => new TopKCategoricalAccuracy(k: k, name: name, dtype: dtype); } } diff --git a/src/TensorFlowNET.Keras/Metrics/TopKCategoricalAccuracy.cs b/src/TensorFlowNET.Keras/Metrics/TopKCategoricalAccuracy.cs new file mode 100644 index 000000000..63e941024 --- /dev/null +++ b/src/TensorFlowNET.Keras/Metrics/TopKCategoricalAccuracy.cs @@ -0,0 +1,12 @@ +namespace Tensorflow.Keras.Metrics; + +public class TopKCategoricalAccuracy : MeanMetricWrapper +{ + public TopKCategoricalAccuracy(int k = 5, string name = "top_k_categorical_accuracy", TF_DataType dtype = TF_DataType.TF_FLOAT) + : base((yt, yp) => metrics_utils.sparse_top_k_categorical_matches( + tf.math.argmax(yt, axis: -1), yp, k), + name: name, + dtype: dtype) + { + } +} diff --git a/src/TensorFlowNET.Keras/Utils/losses_utils.cs b/src/TensorFlowNET.Keras/Utils/losses_utils.cs index 083305954..6de988613 100644 --- a/src/TensorFlowNET.Keras/Utils/losses_utils.cs +++ b/src/TensorFlowNET.Keras/Utils/losses_utils.cs @@ -15,6 +15,7 @@ limitations under the License. ******************************************************************************/ using System; +using System.Xml.Linq; using Tensorflow.Keras.Losses; using static Tensorflow.Binding; @@ -37,15 +38,57 @@ public static Tensor compute_weighted_loss(Tensor losses, Tensor sample_weight = }); } - public static (Tensor, Tensor) squeeze_or_expand_dimensions(Tensor y_pred, Tensor sample_weight) + public static (Tensor, Tensor) squeeze_or_expand_dimensions(Tensor y_pred, Tensor y_true = null, Tensor sample_weight = null) { + var y_pred_shape = y_pred.shape; + var y_pred_rank = y_pred_shape.ndim; + if (y_true != null) + { + var y_true_shape = y_true.shape; + var y_true_rank = y_true_shape.ndim; + if (y_true_rank > -1 && y_pred_rank > -1) + { + if (y_pred_rank - y_true_rank != 1 || y_pred_shape[-1] == 1) + { + (y_true, y_pred) = remove_squeezable_dimensions(y_true, y_pred); + } + } + } + + if (sample_weight == null) + { + return (y_pred, y_true); + } + var weights_shape = sample_weight.shape; var weights_rank = weights_shape.ndim; if (weights_rank == 0) return (y_pred, sample_weight); + + if (y_pred_rank > -1 && weights_rank > -1) + { + if (weights_rank - y_pred_rank == 1) + { + sample_weight = tf.squeeze(sample_weight, -1); + } + else if (y_pred_rank - weights_rank == 1) + { + sample_weight = tf.expand_dims(sample_weight, -1); + } + else + { + return (y_pred, sample_weight); + } + } + throw new NotImplementedException(""); } + public static (Tensor, Tensor) remove_squeezable_dimensions(Tensor labels, Tensor predictions, int expected_rank_diff = 0, string name = null) + { + return (labels, predictions); + } + public static Tensor reduce_weighted_loss(Tensor weighted_losses, string reduction) { if (reduction == ReductionV2.NONE) diff --git a/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs b/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs index bb0107d4e..a35763d09 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs @@ -14,6 +14,26 @@ namespace TensorFlowNET.Keras.UnitTest; [TestClass] public class MetricsTest : EagerModeTestBase { + /// + /// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/TopKCategoricalAccuracy + /// + [TestMethod] + public void TopKCategoricalAccuracy() + { + var y_true = np.array(new[,] { { 0, 0, 1 }, { 0, 1, 0 } }); + var y_pred = np.array(new[,] { { 0.1f, 0.9f, 0.8f }, { 0.05f, 0.95f, 0f } }); + var m = tf.keras.metrics.TopKCategoricalAccuracy(k: 1); + m.update_state(y_true, y_pred); + var r = m.result().numpy(); + Assert.AreEqual(r, 0.5f); + + m.reset_states(); + var weights = np.array(new[] { 0.7f, 0.3f }); + m.update_state(y_true, y_pred, sample_weight: weights); + r = m.result().numpy(); + Assert.AreEqual(r, 0.3f); + } + /// /// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/top_k_categorical_accuracy /// From 217cfd2d4118f1e1188566c9b5e24468ec55e2b0 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Mon, 20 Feb 2023 19:48:06 -0600 Subject: [PATCH 32/52] Add mertic of Recall. --- .../Keras/Metrics/IMetricsApi.cs | 11 ++ .../Operations/array_ops.cs | 3 + src/TensorFlowNET.Keras/GlobalUsing.cs | 4 +- .../Metrics/MeanMetricWrapper.cs | 2 +- src/TensorFlowNET.Keras/Metrics/MetricsApi.cs | 3 + src/TensorFlowNET.Keras/Metrics/Recall.cs | 53 ++++++ src/TensorFlowNET.Keras/Metrics/Reduce.cs | 2 +- .../Metrics/metrics_utils.cs | 171 +++++++++++++++++- src/TensorFlowNET.Keras/Utils/losses_utils.cs | 8 +- .../Metrics/MetricsTest.cs | 20 ++ 10 files changed, 269 insertions(+), 8 deletions(-) create mode 100644 src/TensorFlowNET.Keras/Metrics/Recall.cs diff --git a/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs b/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs index 511b0ef1b..95cc1e600 100644 --- a/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs +++ b/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs @@ -35,4 +35,15 @@ public interface IMetricsApi /// /// IMetricFunc TopKCategoricalAccuracy(int k = 5, string name = "top_k_categorical_accuracy", TF_DataType dtype = TF_DataType.TF_FLOAT); + + /// + /// Computes the recall of the predictions with respect to the labels. + /// + /// + /// + /// + /// + /// + /// + IMetricFunc Recall(float thresholds = 0.5f, int top_k = 1, int class_id = 0, string name = "recall", TF_DataType dtype = TF_DataType.TF_FLOAT); } diff --git a/src/TensorFlowNET.Core/Operations/array_ops.cs b/src/TensorFlowNET.Core/Operations/array_ops.cs index 263509f6f..0e888a0ab 100644 --- a/src/TensorFlowNET.Core/Operations/array_ops.cs +++ b/src/TensorFlowNET.Core/Operations/array_ops.cs @@ -221,6 +221,9 @@ private static TF_DataType _get_dtype_from_nested_lists(IEnumerable list_o case Tensor t: dtype = t.dtype.as_base_dtype(); break; + case int t: + dtype = TF_DataType.TF_INT32; + break; } if (dtype != TF_DataType.DtInvalid) diff --git a/src/TensorFlowNET.Keras/GlobalUsing.cs b/src/TensorFlowNET.Keras/GlobalUsing.cs index 72ff8b289..bc0798ede 100644 --- a/src/TensorFlowNET.Keras/GlobalUsing.cs +++ b/src/TensorFlowNET.Keras/GlobalUsing.cs @@ -1,5 +1,7 @@ global using System; global using System.Collections.Generic; global using System.Text; +global using System.Linq; global using static Tensorflow.Binding; -global using static Tensorflow.KerasApi; \ No newline at end of file +global using static Tensorflow.KerasApi; +global using Tensorflow.NumPy; \ No newline at end of file diff --git a/src/TensorFlowNET.Keras/Metrics/MeanMetricWrapper.cs b/src/TensorFlowNET.Keras/Metrics/MeanMetricWrapper.cs index 2e985b88c..7173aae1d 100644 --- a/src/TensorFlowNET.Keras/Metrics/MeanMetricWrapper.cs +++ b/src/TensorFlowNET.Keras/Metrics/MeanMetricWrapper.cs @@ -18,7 +18,7 @@ public override Tensor update_state(Tensor y_true, Tensor y_pred, Tensor sample_ y_true = math_ops.cast(y_true, _dtype); y_pred = math_ops.cast(y_pred, _dtype); - (y_pred, y_true) = losses_utils.squeeze_or_expand_dimensions(y_pred, y_true: y_true); + (y_pred, y_true, _) = losses_utils.squeeze_or_expand_dimensions(y_pred, y_true: y_true); var matches = _fn(y_true, y_pred); return update_state(matches, sample_weight: sample_weight); diff --git a/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs b/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs index dfccfdbbe..237428328 100644 --- a/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs +++ b/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs @@ -61,5 +61,8 @@ public Tensor top_k_categorical_accuracy(Tensor y_true, Tensor y_pred, int k = 5 public IMetricFunc TopKCategoricalAccuracy(int k = 5, string name = "top_k_categorical_accuracy", TF_DataType dtype = TF_DataType.TF_FLOAT) => new TopKCategoricalAccuracy(k: k, name: name, dtype: dtype); + + public IMetricFunc Recall(float thresholds = 0.5f, int top_k = 1, int class_id = 0, string name = "recall", TF_DataType dtype = TF_DataType.TF_FLOAT) + => new Recall(thresholds: thresholds, top_k: top_k, class_id: class_id, name: name, dtype: dtype); } } diff --git a/src/TensorFlowNET.Keras/Metrics/Recall.cs b/src/TensorFlowNET.Keras/Metrics/Recall.cs new file mode 100644 index 000000000..9b58bf5f7 --- /dev/null +++ b/src/TensorFlowNET.Keras/Metrics/Recall.cs @@ -0,0 +1,53 @@ +namespace Tensorflow.Keras.Metrics; + +public class Recall : Metric +{ + Tensor _thresholds; + int _top_k; + int _class_id; + IVariableV1 true_positives; + IVariableV1 false_negatives; + bool _thresholds_distributed_evenly; + + public Recall(float thresholds = 0.5f, int top_k = 1, int class_id = 0, string name = "recall", TF_DataType dtype = TF_DataType.TF_FLOAT) + : base(name: name, dtype: dtype) + { + _thresholds = constant_op.constant(new float[] { thresholds }); + true_positives = add_weight("true_positives", shape: 1, initializer: tf.initializers.zeros_initializer()); + false_negatives = add_weight("false_negatives", shape: 1, initializer: tf.initializers.zeros_initializer()); + } + + public override Tensor update_state(Tensor y_true, Tensor y_pred, Tensor sample_weight = null) + { + return metrics_utils.update_confusion_matrix_variables( + new Dictionary + { + { "tp", true_positives }, + { "fn", false_negatives }, + }, + y_true, + y_pred, + thresholds: _thresholds, + thresholds_distributed_evenly: _thresholds_distributed_evenly, + top_k: _top_k, + class_id: _class_id, + sample_weight: sample_weight); + } + + public override Tensor result() + { + var result = tf.divide(true_positives.AsTensor(), tf.add(true_positives, false_negatives)); + return _thresholds.size == 1 ? result[0] : result; + } + + public override void reset_states() + { + var num_thresholds = (int)_thresholds.size; + keras.backend.batch_set_value( + new List<(IVariableV1, NDArray)> + { + (true_positives, np.zeros(num_thresholds)), + (false_negatives, np.zeros(num_thresholds)) + }); + } +} diff --git a/src/TensorFlowNET.Keras/Metrics/Reduce.cs b/src/TensorFlowNET.Keras/Metrics/Reduce.cs index f7cdb8f56..8874719de 100644 --- a/src/TensorFlowNET.Keras/Metrics/Reduce.cs +++ b/src/TensorFlowNET.Keras/Metrics/Reduce.cs @@ -27,7 +27,7 @@ public Tensor update_state(Tensor values, Tensor sample_weight = null) { if (sample_weight != null) { - (values, sample_weight) = losses_utils.squeeze_or_expand_dimensions( + (values, _, sample_weight) = losses_utils.squeeze_or_expand_dimensions( values, sample_weight: sample_weight); sample_weight = math_ops.cast(sample_weight, dtype: values.dtype); diff --git a/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs b/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs index de6a8402e..d09b3c722 100644 --- a/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs +++ b/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs @@ -1,4 +1,5 @@ -using Tensorflow.NumPy; +using Tensorflow.Keras.Utils; +using Tensorflow.NumPy; namespace Tensorflow.Keras.Metrics; @@ -36,4 +37,172 @@ public static Tensor sparse_top_k_categorical_matches(Tensor y_true, Tensor y_pr return matches; } + + public static Tensor update_confusion_matrix_variables(Dictionary variables_to_update, + Tensor y_true, + Tensor y_pred, + Tensor thresholds, + int top_k, + int class_id, + Tensor sample_weight = null, + bool multi_label = false, + Tensor label_weights = null, + bool thresholds_distributed_evenly = false) + { + var variable_dtype = variables_to_update.Values.First().dtype; + y_true = tf.cast(y_true, dtype: variable_dtype); + y_pred = tf.cast(y_pred, dtype: variable_dtype); + var num_thresholds = thresholds.shape.dims[0]; + + Tensor one_thresh = null; + if (multi_label) + { + one_thresh = tf.equal(tf.cast(constant_op.constant(1), dtype:tf.int32), + tf.rank(thresholds), + name: "one_set_of_thresholds_cond"); + } + else + { + one_thresh = tf.cast(constant_op.constant(true), dtype: dtypes.@bool); + } + + if (sample_weight == null) + { + (y_pred, y_true, _) = losses_utils.squeeze_or_expand_dimensions(y_pred, y_true); + } + else + { + sample_weight = tf.cast(sample_weight, dtype: variable_dtype); + (y_pred, y_true, sample_weight) = losses_utils.squeeze_or_expand_dimensions(y_pred, + y_true, + sample_weight: sample_weight); + } + + if (thresholds_distributed_evenly) + { + throw new NotImplementedException(); + } + + var pred_shape = tf.shape(y_pred); + var num_predictions = pred_shape[0]; + + Tensor num_labels; + if (y_pred.shape.ndim == 1) + { + num_labels = constant_op.constant(1); + } + else + { + num_labels = tf.reduce_prod(pred_shape["1:"], axis: 0); + } + var thresh_label_tile = tf.where(one_thresh, num_labels, tf.ones(new int[0], dtype: tf.int32)); + + // Reshape predictions and labels, adding a dim for thresholding. + Tensor predictions_extra_dim, labels_extra_dim; + if (multi_label) + { + predictions_extra_dim = tf.expand_dims(y_pred, 0); + labels_extra_dim = tf.expand_dims(tf.cast(y_true, dtype: tf.@bool), 0); + } + + else + { + // Flatten predictions and labels when not multilabel. + predictions_extra_dim = tf.reshape(y_pred, (1, -1)); + labels_extra_dim = tf.reshape(tf.cast(y_true, dtype: tf.@bool), (1, -1)); + } + + // Tile the thresholds for every prediction. + object[] thresh_pretile_shape, thresh_tiles, data_tiles; + + if (multi_label) + { + thresh_pretile_shape = new object[] { num_thresholds, 1, -1 }; + thresh_tiles = new object[] { 1, num_predictions, thresh_label_tile }; + data_tiles = new object[] { num_thresholds, 1, 1 }; + } + else + { + thresh_pretile_shape = new object[] { num_thresholds, -1 }; + thresh_tiles = new object[] { 1, num_predictions * num_labels }; + data_tiles = new object[] { num_thresholds, 1 }; + } + var thresh_tiled = tf.tile(tf.reshape(thresholds, thresh_pretile_shape), tf.stack(thresh_tiles)); + + // Tile the predictions for every threshold. + var preds_tiled = tf.tile(predictions_extra_dim, data_tiles); + + // Compare predictions and threshold. + var pred_is_pos = tf.greater(preds_tiled, thresh_tiled); + + // Tile labels by number of thresholds + var label_is_pos = tf.tile(labels_extra_dim, data_tiles); + + Tensor weights_tiled = null; + + if (sample_weight != null) + { + /*sample_weight = broadcast_weights( + tf.cast(sample_weight, dtype: variable_dtype), y_pred);*/ + weights_tiled = tf.tile( + tf.reshape(sample_weight, thresh_tiles), data_tiles); + } + + if (label_weights != null && !multi_label) + { + throw new NotImplementedException(); + } + + Func weighted_assign_add + = (label, pred, weights, var) => + { + var label_and_pred = tf.cast(tf.logical_and(label, pred), dtype: var.dtype); + if (weights != null) + { + label_and_pred *= tf.cast(weights, dtype: var.dtype); + } + + return var.assign_add(tf.reduce_sum(label_and_pred, 1)); + }; + + + var loop_vars = new Dictionary + { + { "tp", (label_is_pos, pred_is_pos) } + }; + var update_tn = variables_to_update.ContainsKey("tn"); + var update_fp = variables_to_update.ContainsKey("fp"); + var update_fn = variables_to_update.ContainsKey("fn"); + + Tensor pred_is_neg = null; + if (update_fn || update_tn) + { + pred_is_neg = tf.logical_not(pred_is_pos); + loop_vars["fn"] = (label_is_pos, pred_is_neg); + } + + if(update_fp || update_tn) + { + var label_is_neg = tf.logical_not(label_is_pos); + loop_vars["fp"] = (label_is_neg, pred_is_pos); + if (update_tn) + { + loop_vars["tn"] = (label_is_neg, pred_is_neg); + } + } + + var update_ops = new List(); + foreach (var matrix_cond in loop_vars.Keys) + { + var (label, pred) = loop_vars[matrix_cond]; + if (variables_to_update.ContainsKey(matrix_cond)) + { + var op = weighted_assign_add(label, pred, weights_tiled, variables_to_update[matrix_cond]); + update_ops.append(op); + } + } + + tf.group(update_ops.ToArray()); + return null; + } } diff --git a/src/TensorFlowNET.Keras/Utils/losses_utils.cs b/src/TensorFlowNET.Keras/Utils/losses_utils.cs index 6de988613..717acf5e3 100644 --- a/src/TensorFlowNET.Keras/Utils/losses_utils.cs +++ b/src/TensorFlowNET.Keras/Utils/losses_utils.cs @@ -38,7 +38,7 @@ public static Tensor compute_weighted_loss(Tensor losses, Tensor sample_weight = }); } - public static (Tensor, Tensor) squeeze_or_expand_dimensions(Tensor y_pred, Tensor y_true = null, Tensor sample_weight = null) + public static (Tensor, Tensor, Tensor) squeeze_or_expand_dimensions(Tensor y_pred, Tensor y_true = null, Tensor sample_weight = null) { var y_pred_shape = y_pred.shape; var y_pred_rank = y_pred_shape.ndim; @@ -57,13 +57,13 @@ public static (Tensor, Tensor) squeeze_or_expand_dimensions(Tensor y_pred, Tenso if (sample_weight == null) { - return (y_pred, y_true); + return (y_pred, y_true, sample_weight); } var weights_shape = sample_weight.shape; var weights_rank = weights_shape.ndim; if (weights_rank == 0) - return (y_pred, sample_weight); + return (y_pred, y_true, sample_weight); if (y_pred_rank > -1 && weights_rank > -1) { @@ -77,7 +77,7 @@ public static (Tensor, Tensor) squeeze_or_expand_dimensions(Tensor y_pred, Tenso } else { - return (y_pred, sample_weight); + return (y_pred, y_true, sample_weight); } } diff --git a/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs b/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs index a35763d09..84382bb4d 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs @@ -45,4 +45,24 @@ public void top_k_categorical_accuracy() var m = tf.keras.metrics.top_k_categorical_accuracy(y_true, y_pred, k: 3); Assert.AreEqual(m.numpy(), new[] { 1f, 1f }); } + + /// + /// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/Recall + /// + [TestMethod] + public void Recall() + { + var y_true = np.array(new[] { 0, 1, 1, 1 }); + var y_pred = np.array(new[] { 1, 0, 1, 1 }); + var m = tf.keras.metrics.Recall(); + m.update_state(y_true, y_pred); + var r = m.result().numpy(); + Assert.AreEqual(r, 0.6666667f); + + m.reset_states(); + var weights = np.array(new[] { 0f, 0f, 1f, 0f }); + m.update_state(y_true, y_pred, sample_weight: weights); + r = m.result().numpy(); + Assert.AreEqual(r, 1f); + } } From 98919983b14124a22a91f4430c71857e384d7127 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Tue, 21 Feb 2023 21:55:59 -0600 Subject: [PATCH 33/52] Add metrics of Precision. --- src/TensorFlowNET.Core/APIs/tf.math.cs | 11 ++++ .../Keras/Metrics/IMetricsApi.cs | 13 ++++- src/TensorFlowNET.Core/Operations/nn_ops.cs | 4 ++ src/TensorFlowNET.Keras/Metrics/MetricsApi.cs | 5 +- src/TensorFlowNET.Keras/Metrics/Precision.cs | 55 +++++++++++++++++++ .../Metrics/metrics_utils.cs | 22 +++++++- .../Metrics/MetricsTest.cs | 34 ++++++++++++ 7 files changed, 141 insertions(+), 3 deletions(-) create mode 100644 src/TensorFlowNET.Keras/Metrics/Precision.cs diff --git a/src/TensorFlowNET.Core/APIs/tf.math.cs b/src/TensorFlowNET.Core/APIs/tf.math.cs index 7d3f6eff9..0191f8d60 100644 --- a/src/TensorFlowNET.Core/APIs/tf.math.cs +++ b/src/TensorFlowNET.Core/APIs/tf.math.cs @@ -39,6 +39,17 @@ public Tensor erf(Tensor x, string name = null) public Tensor sum(Tensor x, Axis? axis = null, string name = null) => math_ops.reduce_sum(x, axis: axis, name: name); + /// + /// Finds values and indices of the `k` largest entries for the last dimension. + /// + /// + /// + /// + /// + /// + public Tensors top_k(Tensor input, int k, bool sorted = true, string name = null) + => nn_ops.top_kv2(input, k, sorted: sorted, name: name); + public Tensor in_top_k(Tensor predictions, Tensor targets, int k, string name = "InTopK") => nn_ops.in_top_k(predictions, targets, k, name); diff --git a/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs b/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs index 95cc1e600..e27c198db 100644 --- a/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs +++ b/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs @@ -36,6 +36,17 @@ public interface IMetricsApi /// IMetricFunc TopKCategoricalAccuracy(int k = 5, string name = "top_k_categorical_accuracy", TF_DataType dtype = TF_DataType.TF_FLOAT); + /// + /// Computes the precision of the predictions with respect to the labels. + /// + /// + /// + /// + /// + /// + /// + IMetricFunc Precision(float thresholds = 0.5f, int top_k = 0, int class_id = 0, string name = "recall", TF_DataType dtype = TF_DataType.TF_FLOAT); + /// /// Computes the recall of the predictions with respect to the labels. /// @@ -45,5 +56,5 @@ public interface IMetricsApi /// /// /// - IMetricFunc Recall(float thresholds = 0.5f, int top_k = 1, int class_id = 0, string name = "recall", TF_DataType dtype = TF_DataType.TF_FLOAT); + IMetricFunc Recall(float thresholds = 0.5f, int top_k = 0, int class_id = 0, string name = "recall", TF_DataType dtype = TF_DataType.TF_FLOAT); } diff --git a/src/TensorFlowNET.Core/Operations/nn_ops.cs b/src/TensorFlowNET.Core/Operations/nn_ops.cs index 307b1f8af..5877d234d 100644 --- a/src/TensorFlowNET.Core/Operations/nn_ops.cs +++ b/src/TensorFlowNET.Core/Operations/nn_ops.cs @@ -109,6 +109,10 @@ private static Tensor _get_noise_shape(Tensor x, Tensor noise_shape) return noise_shape; } + public static Tensors top_kv2(Tensor input, int k, bool sorted = true, string name = null) + => tf.Context.ExecuteOp("TopKV2", name, new ExecuteOpArgs(input, k) + .SetAttributes(new { sorted })); + public static Tensor in_top_k(Tensor predictions, Tensor targets, int k, string name = null) { return tf_with(ops.name_scope(name, "in_top_k"), delegate diff --git a/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs b/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs index 237428328..74db16660 100644 --- a/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs +++ b/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs @@ -62,7 +62,10 @@ public Tensor top_k_categorical_accuracy(Tensor y_true, Tensor y_pred, int k = 5 public IMetricFunc TopKCategoricalAccuracy(int k = 5, string name = "top_k_categorical_accuracy", TF_DataType dtype = TF_DataType.TF_FLOAT) => new TopKCategoricalAccuracy(k: k, name: name, dtype: dtype); - public IMetricFunc Recall(float thresholds = 0.5f, int top_k = 1, int class_id = 0, string name = "recall", TF_DataType dtype = TF_DataType.TF_FLOAT) + public IMetricFunc Precision(float thresholds = 0.5f, int top_k = 0, int class_id = 0, string name = "precision", TF_DataType dtype = TF_DataType.TF_FLOAT) + => new Precision(thresholds: thresholds, top_k: top_k, class_id: class_id, name: name, dtype: dtype); + + public IMetricFunc Recall(float thresholds = 0.5f, int top_k = 0, int class_id = 0, string name = "recall", TF_DataType dtype = TF_DataType.TF_FLOAT) => new Recall(thresholds: thresholds, top_k: top_k, class_id: class_id, name: name, dtype: dtype); } } diff --git a/src/TensorFlowNET.Keras/Metrics/Precision.cs b/src/TensorFlowNET.Keras/Metrics/Precision.cs new file mode 100644 index 000000000..a01773e0e --- /dev/null +++ b/src/TensorFlowNET.Keras/Metrics/Precision.cs @@ -0,0 +1,55 @@ +namespace Tensorflow.Keras.Metrics; + +public class Precision : Metric +{ + Tensor _thresholds; + int _top_k; + int _class_id; + IVariableV1 true_positives; + IVariableV1 false_positives; + bool _thresholds_distributed_evenly; + + public Precision(float thresholds = 0.5f, int top_k = 0, int class_id = 0, string name = "recall", TF_DataType dtype = TF_DataType.TF_FLOAT) + : base(name: name, dtype: dtype) + { + _thresholds = constant_op.constant(new float[] { thresholds }); + _top_k = top_k; + _class_id = class_id; + true_positives = add_weight("true_positives", shape: 1, initializer: tf.initializers.zeros_initializer()); + false_positives = add_weight("false_positives", shape: 1, initializer: tf.initializers.zeros_initializer()); + } + + public override Tensor update_state(Tensor y_true, Tensor y_pred, Tensor sample_weight = null) + { + return metrics_utils.update_confusion_matrix_variables( + new Dictionary + { + { "tp", true_positives }, + { "fp", false_positives }, + }, + y_true, + y_pred, + thresholds: _thresholds, + thresholds_distributed_evenly: _thresholds_distributed_evenly, + top_k: _top_k, + class_id: _class_id, + sample_weight: sample_weight); + } + + public override Tensor result() + { + var result = tf.divide(true_positives.AsTensor(), tf.add(true_positives, false_positives)); + return _thresholds.size == 1 ? result[0] : result; + } + + public override void reset_states() + { + var num_thresholds = (int)_thresholds.size; + keras.backend.batch_set_value( + new List<(IVariableV1, NDArray)> + { + (true_positives, np.zeros(num_thresholds)), + (false_positives, np.zeros(num_thresholds)) + }); + } +} diff --git a/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs b/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs index d09b3c722..0251462ee 100644 --- a/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs +++ b/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs @@ -78,6 +78,17 @@ public static Tensor update_confusion_matrix_variables(Dictionary 0) + { + y_pred = _filter_top_k(y_pred, top_k); + } + + if (class_id > 0) + { + y_true = y_true[Slice.All, class_id]; + y_pred = y_pred[Slice.All, class_id]; + } + if (thresholds_distributed_evenly) { throw new NotImplementedException(); @@ -204,5 +215,14 @@ Func weighted_assign_ad tf.group(update_ops.ToArray()); return null; - } + } + + private static Tensor _filter_top_k(Tensor x, int k) + { + var NEG_INF = -1e10; + var (_, top_k_idx) = tf.math.top_k(x, k, sorted: false); + var top_k_mask = tf.reduce_sum( + tf.one_hot(top_k_idx, (int)x.shape[-1], axis: -1), axis: -2); + return x * top_k_mask + NEG_INF * (1 - top_k_mask); + } } diff --git a/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs b/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs index 84382bb4d..f3ba2e93b 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs @@ -46,6 +46,40 @@ public void top_k_categorical_accuracy() Assert.AreEqual(m.numpy(), new[] { 1f, 1f }); } + /// + /// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/Precision + /// + [TestMethod] + public void Precision() + { + var y_true = np.array(new[] { 0, 1, 1, 1 }); + var y_pred = np.array(new[] { 1, 0, 1, 1 }); + var m = tf.keras.metrics.Precision(); + m.update_state(y_true, y_pred); + var r = m.result().numpy(); + Assert.AreEqual(r, 0.6666667f); + + m.reset_states(); + var weights = np.array(new[] { 0f, 0f, 1f, 0f }); + m.update_state(y_true, y_pred, sample_weight: weights); + r = m.result().numpy(); + Assert.AreEqual(r, 1f); + + // With top_k=2, it will calculate precision over y_true[:2] + // and y_pred[:2] + m = tf.keras.metrics.Precision(top_k: 2); + m.update_state(np.array(new[] { 0, 0, 1, 1 }), np.array(new[] { 1, 1, 1, 1 })); + r = m.result().numpy(); + Assert.AreEqual(r, 0f); + + // With top_k=4, it will calculate precision over y_true[:4] + // and y_pred[:4] + m = tf.keras.metrics.Precision(top_k: 4); + m.update_state(np.array(new[] { 0, 0, 1, 1 }), np.array(new[] { 1, 1, 1, 1 })); + r = m.result().numpy(); + Assert.AreEqual(r, 0.5f); + } + /// /// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/Recall /// From 9e877d1c1532883b978e085cc71fc1136593a30f Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 22 Feb 2023 10:44:14 -0600 Subject: [PATCH 34/52] Add metrics of BinaryAccuracy, CategoricalAccuracy, CategoricalCrossentropy. --- .../Keras/Metrics/IMetricFunc.cs | 1 + .../Keras/Metrics/IMetricsApi.cs | 47 +++++++++++++-- .../Engine/MetricsContainer.cs | 24 +++++--- .../Engine/Model.Compile.cs | 22 +++++++ .../Engine/Model.Metrics.cs | 4 +- .../Metrics/BinaryAccuracy.cs | 11 ++++ .../Metrics/CategoricalAccuracy.cs | 12 ++++ .../Metrics/CategoricalCrossentropy.cs | 16 +++++ src/TensorFlowNET.Keras/Metrics/MetricsApi.cs | 21 +++++++ .../Metrics/metrics_utils.cs | 40 ++++++++++++- src/TensorFlowNET.Keras/Utils/losses_utils.cs | 6 +- .../Metrics/MetricsTest.cs | 60 +++++++++++++++++++ 12 files changed, 244 insertions(+), 20 deletions(-) create mode 100644 src/TensorFlowNET.Keras/Metrics/BinaryAccuracy.cs create mode 100644 src/TensorFlowNET.Keras/Metrics/CategoricalAccuracy.cs create mode 100644 src/TensorFlowNET.Keras/Metrics/CategoricalCrossentropy.cs diff --git a/src/TensorFlowNET.Core/Keras/Metrics/IMetricFunc.cs b/src/TensorFlowNET.Core/Keras/Metrics/IMetricFunc.cs index 1867d6375..930afa0b0 100644 --- a/src/TensorFlowNET.Core/Keras/Metrics/IMetricFunc.cs +++ b/src/TensorFlowNET.Core/Keras/Metrics/IMetricFunc.cs @@ -2,6 +2,7 @@ public interface IMetricFunc { + string Name { get; } /// /// Accumulates metric statistics. /// diff --git a/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs b/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs index e27c198db..759463035 100644 --- a/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs +++ b/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs @@ -5,6 +5,10 @@ public interface IMetricsApi Tensor binary_accuracy(Tensor y_true, Tensor y_pred); Tensor categorical_accuracy(Tensor y_true, Tensor y_pred); + Tensor categorical_crossentropy(Tensor y_true, Tensor y_pred, + bool from_logits = false, + float label_smoothing = 0f, + Axis? axis = null); Tensor mean_absolute_error(Tensor y_true, Tensor y_pred); @@ -27,14 +31,39 @@ public interface IMetricsApi /// Tensor top_k_categorical_accuracy(Tensor y_true, Tensor y_pred, int k = 5); + /// + /// Calculates how often predictions match binary labels. + /// + /// + IMetricFunc BinaryAccuracy(string name = "binary_accuracy", + TF_DataType dtype = TF_DataType.TF_FLOAT, + float threshold = 05f); + + /// + /// Calculates how often predictions match one-hot labels. + /// + /// + IMetricFunc CategoricalCrossentropy(string name = "categorical_crossentropy", + TF_DataType dtype = TF_DataType.TF_FLOAT, + bool from_logits = false, + float label_smoothing = 0f, + Axis? axis = null); + + /// + /// Computes the crossentropy metric between the labels and predictions. + /// + /// + IMetricFunc CategoricalAccuracy(string name = "categorical_accuracy", + TF_DataType dtype = TF_DataType.TF_FLOAT); + /// /// Computes how often targets are in the top K predictions. /// - /// - /// /// /// - IMetricFunc TopKCategoricalAccuracy(int k = 5, string name = "top_k_categorical_accuracy", TF_DataType dtype = TF_DataType.TF_FLOAT); + IMetricFunc TopKCategoricalAccuracy(int k = 5, + string name = "top_k_categorical_accuracy", + TF_DataType dtype = TF_DataType.TF_FLOAT); /// /// Computes the precision of the predictions with respect to the labels. @@ -45,7 +74,11 @@ public interface IMetricsApi /// /// /// - IMetricFunc Precision(float thresholds = 0.5f, int top_k = 0, int class_id = 0, string name = "recall", TF_DataType dtype = TF_DataType.TF_FLOAT); + IMetricFunc Precision(float thresholds = 0.5f, + int top_k = 0, + int class_id = 0, + string name = "recall", + TF_DataType dtype = TF_DataType.TF_FLOAT); /// /// Computes the recall of the predictions with respect to the labels. @@ -56,5 +89,9 @@ public interface IMetricsApi /// /// /// - IMetricFunc Recall(float thresholds = 0.5f, int top_k = 0, int class_id = 0, string name = "recall", TF_DataType dtype = TF_DataType.TF_FLOAT); + IMetricFunc Recall(float thresholds = 0.5f, + int top_k = 0, + int class_id = 0, + string name = "recall", + TF_DataType dtype = TF_DataType.TF_FLOAT); } diff --git a/src/TensorFlowNET.Keras/Engine/MetricsContainer.cs b/src/TensorFlowNET.Keras/Engine/MetricsContainer.cs index 5eb05eaa7..ee6384107 100644 --- a/src/TensorFlowNET.Keras/Engine/MetricsContainer.cs +++ b/src/TensorFlowNET.Keras/Engine/MetricsContainer.cs @@ -9,15 +9,21 @@ namespace Tensorflow.Keras.Engine { public class MetricsContainer : Container { - string[] _user_metrics; - string[] _metric_names; - Metric[] _metrics; - List _metrics_in_order; + IMetricFunc[] _user_metrics = new IMetricFunc[0]; + string[] _metric_names = new string[0]; + Metric[] _metrics = new Metric[0]; + List _metrics_in_order = new List(); - public MetricsContainer(string[] metrics, string[] output_names = null) + public MetricsContainer(IMetricFunc[] metrics, string[] output_names = null) : base(output_names) { _user_metrics = metrics; + _built = false; + } + + public MetricsContainer(string[] metrics, string[] output_names = null) + : base(output_names) + { _metric_names = metrics; _built = false; } @@ -46,9 +52,11 @@ void _set_metric_names() void _create_ordered_metrics() { - _metrics_in_order = new List(); foreach (var m in _metrics) _metrics_in_order.append(m); + + foreach(var m in _user_metrics) + _metrics_in_order.append(m); } Metric[] _get_metric_objects(string[] metrics, Tensor y_t, Tensor y_p) @@ -56,7 +64,7 @@ Metric[] _get_metric_objects(string[] metrics, Tensor y_t, Tensor y_p) return metrics.Select(x => _get_metric_object(x, y_t, y_p)).ToArray(); } - Metric _get_metric_object(string metric, Tensor y_t, Tensor y_p) + public Metric _get_metric_object(string metric, Tensor y_t, Tensor y_p) { Func metric_obj = null; if (metric == "accuracy" || metric == "acc") @@ -94,7 +102,7 @@ Metric _get_metric_object(string metric, Tensor y_t, Tensor y_p) return new MeanMetricWrapper(metric_obj, metric); } - public IEnumerable metrics + public IEnumerable metrics { get { diff --git a/src/TensorFlowNET.Keras/Engine/Model.Compile.cs b/src/TensorFlowNET.Keras/Engine/Model.Compile.cs index 7b051f1d0..3d99129b0 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Compile.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Compile.cs @@ -1,6 +1,7 @@ using System; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Losses; +using Tensorflow.Keras.Metrics; using Tensorflow.Keras.Optimizers; namespace Tensorflow.Keras.Engine @@ -31,6 +32,27 @@ public void compile(OptimizerV2 optimizer = null, _is_compiled = true; } + public void compile(OptimizerV2 optimizer = null, + ILossFunc loss = null, + IMetricFunc[] metrics = null) + { + this.optimizer = optimizer ?? new RMSprop(new RMSpropArgs + { + }); + + this.loss = loss ?? new MeanSquaredError(); + + compiled_loss = new LossesContainer(loss, output_names: output_names); + compiled_metrics = new MetricsContainer(metrics, output_names: output_names); + + int experimental_steps_per_execution = 1; + _configure_steps_per_execution(experimental_steps_per_execution); + + // Initialize cache attrs. + _reset_compile_cache(); + _is_compiled = true; + } + public void compile(string optimizer, string loss, string[] metrics) { var _optimizer = optimizer switch diff --git a/src/TensorFlowNET.Keras/Engine/Model.Metrics.cs b/src/TensorFlowNET.Keras/Engine/Model.Metrics.cs index 214b99345..0e33b14e3 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Metrics.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Metrics.cs @@ -5,11 +5,11 @@ namespace Tensorflow.Keras.Engine { public partial class Model { - public IEnumerable metrics + public IEnumerable metrics { get { - var _metrics = new List(); + var _metrics = new List(); if (_is_compiled) { diff --git a/src/TensorFlowNET.Keras/Metrics/BinaryAccuracy.cs b/src/TensorFlowNET.Keras/Metrics/BinaryAccuracy.cs new file mode 100644 index 000000000..2977588e9 --- /dev/null +++ b/src/TensorFlowNET.Keras/Metrics/BinaryAccuracy.cs @@ -0,0 +1,11 @@ +namespace Tensorflow.Keras.Metrics; + +public class BinaryAccuracy : MeanMetricWrapper +{ + public BinaryAccuracy(string name = "binary_accuracy", TF_DataType dtype = TF_DataType.TF_FLOAT, float threshold = 0.5f) + : base((yt, yp) => metrics_utils.binary_matches(yt, yp), + name: name, + dtype: dtype) + { + } +} diff --git a/src/TensorFlowNET.Keras/Metrics/CategoricalAccuracy.cs b/src/TensorFlowNET.Keras/Metrics/CategoricalAccuracy.cs new file mode 100644 index 000000000..d15cf26c5 --- /dev/null +++ b/src/TensorFlowNET.Keras/Metrics/CategoricalAccuracy.cs @@ -0,0 +1,12 @@ +namespace Tensorflow.Keras.Metrics; + +public class CategoricalAccuracy : MeanMetricWrapper +{ + public CategoricalAccuracy(string name = "categorical_accuracy", TF_DataType dtype = TF_DataType.TF_FLOAT) + : base((yt, yp) => metrics_utils.sparse_categorical_matches( + tf.math.argmax(yt, axis: -1), yp), + name: name, + dtype: dtype) + { + } +} diff --git a/src/TensorFlowNET.Keras/Metrics/CategoricalCrossentropy.cs b/src/TensorFlowNET.Keras/Metrics/CategoricalCrossentropy.cs new file mode 100644 index 000000000..95720c413 --- /dev/null +++ b/src/TensorFlowNET.Keras/Metrics/CategoricalCrossentropy.cs @@ -0,0 +1,16 @@ +namespace Tensorflow.Keras.Metrics; + +public class CategoricalCrossentropy : MeanMetricWrapper +{ + public CategoricalCrossentropy(string name = "categorical_crossentropy", + TF_DataType dtype = TF_DataType.TF_FLOAT, + bool from_logits = false, + float label_smoothing = 0f, + Axis? axis = null) + : base((yt, yp) => keras.metrics.categorical_crossentropy( + yt, yp, from_logits: from_logits, label_smoothing: label_smoothing, axis: axis ?? -1), + name: name, + dtype: dtype) + { + } +} diff --git a/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs b/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs index 74db16660..fcd0516bd 100644 --- a/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs +++ b/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs @@ -15,6 +15,18 @@ public Tensor categorical_accuracy(Tensor y_true, Tensor y_pred) return math_ops.cast(eql, TF_DataType.TF_FLOAT); } + public Tensor categorical_crossentropy(Tensor y_true, Tensor y_pred, bool from_logits = false, float label_smoothing = 0, Axis? axis = null) + { + y_true = tf.cast(y_true, y_pred.dtype); + // var label_smoothing_tensor = tf.convert_to_tensor(label_smoothing, dtype: y_pred.dtype); + if (label_smoothing > 0) + { + var num_classes = tf.cast(tf.shape(y_true)[-1], y_pred.dtype); + y_true = y_true * (1.0 - label_smoothing) + (label_smoothing / num_classes); + } + return keras.backend.categorical_crossentropy(y_true, y_pred, from_logits: from_logits, axis: axis); + } + /// /// Calculates how often predictions matches integer labels. /// @@ -59,6 +71,15 @@ public Tensor top_k_categorical_accuracy(Tensor y_true, Tensor y_pred, int k = 5 ); } + public IMetricFunc BinaryAccuracy(string name = "binary_accuracy", TF_DataType dtype = TF_DataType.TF_FLOAT, float threshold = 5) + => new BinaryAccuracy(); + + public IMetricFunc CategoricalAccuracy(string name = "categorical_accuracy", TF_DataType dtype = TF_DataType.TF_FLOAT) + => new CategoricalAccuracy(name: name, dtype: dtype); + + public IMetricFunc CategoricalCrossentropy(string name = "categorical_crossentropy", TF_DataType dtype = TF_DataType.TF_FLOAT, bool from_logits = false, float label_smoothing = 0, Axis? axis = null) + => new CategoricalCrossentropy(); + public IMetricFunc TopKCategoricalAccuracy(int k = 5, string name = "top_k_categorical_accuracy", TF_DataType dtype = TF_DataType.TF_FLOAT) => new TopKCategoricalAccuracy(k: k, name: name, dtype: dtype); diff --git a/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs b/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs index 0251462ee..0f523e7e1 100644 --- a/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs +++ b/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs @@ -1,10 +1,48 @@ using Tensorflow.Keras.Utils; -using Tensorflow.NumPy; namespace Tensorflow.Keras.Metrics; public class metrics_utils { + public static Tensor binary_matches(Tensor y_true, Tensor y_pred, float threshold = 0.5f) + { + y_pred = tf.cast(y_pred > threshold, y_pred.dtype); + return tf.cast(tf.equal(y_true, y_pred), keras.backend.floatx()); + } + + /// + /// Creates float Tensor, 1.0 for label-prediction match, 0.0 for mismatch. + /// + /// + /// + /// + public static Tensor sparse_categorical_matches(Tensor y_true, Tensor y_pred) + { + var reshape_matches = false; + var y_true_rank = y_true.shape.ndim; + var y_pred_rank = y_pred.shape.ndim; + var y_true_org_shape = tf.shape(y_true); + + if (y_true_rank > -1 && y_pred_rank > -1 && y_true.ndim == y_pred.ndim ) + { + reshape_matches = true; + y_true = tf.squeeze(y_true, new Shape(-1)); + } + y_pred = tf.math.argmax(y_pred, axis: -1); + + var matches = tf.cast( + tf.equal(y_true, y_pred), + dtype: keras.backend.floatx() + ); + + if (reshape_matches) + { + return tf.reshape(matches, shape: y_true_org_shape); + } + + return matches; + } + public static Tensor sparse_top_k_categorical_matches(Tensor y_true, Tensor y_pred, int k = 5) { var reshape_matches = false; diff --git a/src/TensorFlowNET.Keras/Utils/losses_utils.cs b/src/TensorFlowNET.Keras/Utils/losses_utils.cs index 717acf5e3..9ba40ca04 100644 --- a/src/TensorFlowNET.Keras/Utils/losses_utils.cs +++ b/src/TensorFlowNET.Keras/Utils/losses_utils.cs @@ -75,10 +75,8 @@ public static (Tensor, Tensor, Tensor) squeeze_or_expand_dimensions(Tensor y_pre { sample_weight = tf.expand_dims(sample_weight, -1); } - else - { - return (y_pred, y_true, sample_weight); - } + + return (y_pred, y_true, sample_weight); } throw new NotImplementedException(""); diff --git a/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs b/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs index f3ba2e93b..9389af96f 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs @@ -14,6 +14,66 @@ namespace TensorFlowNET.Keras.UnitTest; [TestClass] public class MetricsTest : EagerModeTestBase { + /// + /// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/BinaryAccuracy + /// + [TestMethod] + public void BinaryAccuracy() + { + var y_true = np.array(new[,] { { 1 }, { 1 },{ 0 }, { 0 } }); + var y_pred = np.array(new[,] { { 0.98f }, { 1f }, { 0f }, { 0.6f } }); + var m = tf.keras.metrics.BinaryAccuracy(); + /*m.update_state(y_true, y_pred); + var r = m.result().numpy(); + Assert.AreEqual(r, 0.75f); + + m.reset_states();*/ + var weights = np.array(new[] { 1f, 0f, 0f, 1f }); + m.update_state(y_true, y_pred, sample_weight: weights); + var r = m.result().numpy(); + Assert.AreEqual(r, 0.5f); + } + + /// + /// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/CategoricalAccuracy + /// + [TestMethod] + public void CategoricalAccuracy() + { + var y_true = np.array(new[,] { { 0, 0, 1 }, { 0, 1, 0 } }); + var y_pred = np.array(new[,] { { 0.1f, 0.9f, 0.8f }, { 0.05f, 0.95f, 0f } }); + var m = tf.keras.metrics.CategoricalAccuracy(); + m.update_state(y_true, y_pred); + var r = m.result().numpy(); + Assert.AreEqual(r, 0.5f); + + m.reset_states(); + var weights = np.array(new[] { 0.7f, 0.3f }); + m.update_state(y_true, y_pred, sample_weight: weights); + r = m.result().numpy(); + Assert.AreEqual(r, 0.3f); + } + + /// + /// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/CategoricalCrossentropy + /// + [TestMethod] + public void CategoricalCrossentropy() + { + var y_true = np.array(new[,] { { 0, 1, 0 }, { 0, 0, 1 } }); + var y_pred = np.array(new[,] { { 0.05f, 0.95f, 0f }, { 0.1f, 0.8f, 0.1f } }); + var m = tf.keras.metrics.CategoricalCrossentropy(); + m.update_state(y_true, y_pred); + var r = m.result().numpy(); + Assert.AreEqual(r, 1.1769392f); + + m.reset_states(); + var weights = np.array(new[] { 0.3f, 0.7f }); + m.update_state(y_true, y_pred, sample_weight: weights); + r = m.result().numpy(); + Assert.AreEqual(r, 1.6271976f); + } + /// /// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/TopKCategoricalAccuracy /// From 77eb6f5e71fbdd41cf2185093e9ef53583fe801a Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Wed, 22 Feb 2023 11:14:55 -0600 Subject: [PATCH 35/52] Add metrics of Accuracy and CosineSimilarity. --- src/TensorFlowNET.Core/APIs/tf.linalg.cs | 6 +++ .../Keras/Metrics/IMetricsApi.cs | 15 ++++++ src/TensorFlowNET.Keras/Metrics/Accuracy.cs | 11 +++++ .../Metrics/CosineSimilarity.cs | 11 +++++ src/TensorFlowNET.Keras/Metrics/MetricsApi.cs | 6 +++ .../Metrics/metrics_utils.cs | 14 ++++++ .../Metrics/MetricsTest.cs | 46 +++++++++++++++++-- 7 files changed, 106 insertions(+), 3 deletions(-) create mode 100644 src/TensorFlowNET.Keras/Metrics/Accuracy.cs create mode 100644 src/TensorFlowNET.Keras/Metrics/CosineSimilarity.cs diff --git a/src/TensorFlowNET.Core/APIs/tf.linalg.cs b/src/TensorFlowNET.Core/APIs/tf.linalg.cs index 10c09d994..32f64ec35 100644 --- a/src/TensorFlowNET.Core/APIs/tf.linalg.cs +++ b/src/TensorFlowNET.Core/APIs/tf.linalg.cs @@ -54,6 +54,12 @@ public Tensor inv(Tensor input, bool adjoint = false, string name = null) public Tensor global_norm(Tensor[] t_list, string name = null) => clip_ops.global_norm(t_list, name: name); + public Tensor l2_normalize(Tensor x, + int axis = 0, + float epsilon = 1e-12f, + string name = null) + => nn_impl.l2_normalize(x, axis: axis, epsilon: constant_op.constant(epsilon), name: name); + public Tensor lstsq(Tensor matrix, Tensor rhs, NDArray l2_regularizer = null, bool fast = true, string name = null) => ops.matrix_solve_ls(matrix, rhs, l2_regularizer: l2_regularizer, fast: fast, name: name); diff --git a/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs b/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs index 759463035..e4575620a 100644 --- a/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs +++ b/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs @@ -31,6 +31,13 @@ Tensor categorical_crossentropy(Tensor y_true, Tensor y_pred, /// Tensor top_k_categorical_accuracy(Tensor y_true, Tensor y_pred, int k = 5); + /// + /// Calculates how often predictions equal labels. + /// + /// + IMetricFunc Accuracy(string name = "accuracy", + TF_DataType dtype = TF_DataType.TF_FLOAT); + /// /// Calculates how often predictions match binary labels. /// @@ -56,6 +63,14 @@ IMetricFunc CategoricalCrossentropy(string name = "categorical_crossentropy", IMetricFunc CategoricalAccuracy(string name = "categorical_accuracy", TF_DataType dtype = TF_DataType.TF_FLOAT); + /// + /// Computes the cosine similarity between the labels and predictions. + /// + /// + IMetricFunc CosineSimilarity(string name = "cosine_similarity", + TF_DataType dtype = TF_DataType.TF_FLOAT, + Axis? axis = null); + /// /// Computes how often targets are in the top K predictions. /// diff --git a/src/TensorFlowNET.Keras/Metrics/Accuracy.cs b/src/TensorFlowNET.Keras/Metrics/Accuracy.cs new file mode 100644 index 000000000..93a724679 --- /dev/null +++ b/src/TensorFlowNET.Keras/Metrics/Accuracy.cs @@ -0,0 +1,11 @@ +namespace Tensorflow.Keras.Metrics; + +public class Accuracy : MeanMetricWrapper +{ + public Accuracy(string name = "accuracy", TF_DataType dtype = TF_DataType.TF_FLOAT) + : base((yt, yp) => metrics_utils.accuracy(yt, yp), + name: name, + dtype: dtype) + { + } +} diff --git a/src/TensorFlowNET.Keras/Metrics/CosineSimilarity.cs b/src/TensorFlowNET.Keras/Metrics/CosineSimilarity.cs new file mode 100644 index 000000000..2a26bcdfe --- /dev/null +++ b/src/TensorFlowNET.Keras/Metrics/CosineSimilarity.cs @@ -0,0 +1,11 @@ +namespace Tensorflow.Keras.Metrics; + +public class CosineSimilarity : MeanMetricWrapper +{ + public CosineSimilarity(string name = "cosine_similarity", TF_DataType dtype = TF_DataType.TF_FLOAT, Axis? axis = null) + : base((yt, yp) => metrics_utils.cosine_similarity(yt, yp, axis: axis ?? -1), + name: name, + dtype: dtype) + { + } +} diff --git a/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs b/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs index fcd0516bd..e207d27da 100644 --- a/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs +++ b/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs @@ -71,6 +71,9 @@ public Tensor top_k_categorical_accuracy(Tensor y_true, Tensor y_pred, int k = 5 ); } + public IMetricFunc Accuracy(string name = "accuracy", TF_DataType dtype = TF_DataType.TF_FLOAT) + => new Accuracy(name: name, dtype: dtype); + public IMetricFunc BinaryAccuracy(string name = "binary_accuracy", TF_DataType dtype = TF_DataType.TF_FLOAT, float threshold = 5) => new BinaryAccuracy(); @@ -80,6 +83,9 @@ public IMetricFunc CategoricalAccuracy(string name = "categorical_accuracy", TF_ public IMetricFunc CategoricalCrossentropy(string name = "categorical_crossentropy", TF_DataType dtype = TF_DataType.TF_FLOAT, bool from_logits = false, float label_smoothing = 0, Axis? axis = null) => new CategoricalCrossentropy(); + public IMetricFunc CosineSimilarity(string name = "cosine_similarity", TF_DataType dtype = TF_DataType.TF_FLOAT, Axis? axis = null) + => new CosineSimilarity(name: name, dtype: dtype, axis: axis ?? -1); + public IMetricFunc TopKCategoricalAccuracy(int k = 5, string name = "top_k_categorical_accuracy", TF_DataType dtype = TF_DataType.TF_FLOAT) => new TopKCategoricalAccuracy(k: k, name: name, dtype: dtype); diff --git a/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs b/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs index 0f523e7e1..f4bfc3da4 100644 --- a/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs +++ b/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs @@ -4,12 +4,26 @@ namespace Tensorflow.Keras.Metrics; public class metrics_utils { + public static Tensor accuracy(Tensor y_true, Tensor y_pred) + { + if (y_true.dtype != y_pred.dtype) + y_pred = tf.cast(y_pred, y_true.dtype); + return tf.cast(tf.equal(y_true, y_pred), keras.backend.floatx()); + } + public static Tensor binary_matches(Tensor y_true, Tensor y_pred, float threshold = 0.5f) { y_pred = tf.cast(y_pred > threshold, y_pred.dtype); return tf.cast(tf.equal(y_true, y_pred), keras.backend.floatx()); } + public static Tensor cosine_similarity(Tensor y_true, Tensor y_pred, Axis? axis = null) + { + y_true = tf.linalg.l2_normalize(y_true, axis: axis ?? -1); + y_pred = tf.linalg.l2_normalize(y_pred, axis: axis ?? -1); + return tf.reduce_sum(y_true * y_pred, axis: axis ?? -1); + } + /// /// Creates float Tensor, 1.0 for label-prediction match, 0.0 for mismatch. /// diff --git a/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs b/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs index 9389af96f..90be51bd4 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs @@ -14,6 +14,26 @@ namespace TensorFlowNET.Keras.UnitTest; [TestClass] public class MetricsTest : EagerModeTestBase { + /// + /// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/Accuracy + /// + [TestMethod] + public void Accuracy() + { + var y_true = np.array(new[,] { { 1 }, { 2 }, { 3 }, { 4 } }); + var y_pred = np.array(new[,] { { 0f }, { 2f }, { 3f }, { 4f } }); + var m = tf.keras.metrics.Accuracy(); + m.update_state(y_true, y_pred); + var r = m.result().numpy(); + Assert.AreEqual(r, 0.75f); + + m.reset_states(); + var weights = np.array(new[] { 1f, 1f, 0f, 0f }); + m.update_state(y_true, y_pred, sample_weight: weights); + r = m.result().numpy(); + Assert.AreEqual(r, 0.5f); + } + /// /// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/BinaryAccuracy /// @@ -23,14 +43,14 @@ public void BinaryAccuracy() var y_true = np.array(new[,] { { 1 }, { 1 },{ 0 }, { 0 } }); var y_pred = np.array(new[,] { { 0.98f }, { 1f }, { 0f }, { 0.6f } }); var m = tf.keras.metrics.BinaryAccuracy(); - /*m.update_state(y_true, y_pred); + m.update_state(y_true, y_pred); var r = m.result().numpy(); Assert.AreEqual(r, 0.75f); - m.reset_states();*/ + m.reset_states(); var weights = np.array(new[] { 1f, 0f, 0f, 1f }); m.update_state(y_true, y_pred, sample_weight: weights); - var r = m.result().numpy(); + r = m.result().numpy(); Assert.AreEqual(r, 0.5f); } @@ -74,6 +94,26 @@ public void CategoricalCrossentropy() Assert.AreEqual(r, 1.6271976f); } + /// + /// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/CosineSimilarity + /// + [TestMethod] + public void CosineSimilarity() + { + var y_true = np.array(new[,] { { 0, 1 }, { 1, 1 } }); + var y_pred = np.array(new[,] { { 1f, 0f }, { 1f, 1f } }); + var m = tf.keras.metrics.CosineSimilarity(axis: 1); + m.update_state(y_true, y_pred); + var r = m.result().numpy(); + Assert.AreEqual(r, 0.49999997f); + + m.reset_states(); + var weights = np.array(new[] { 0.3f, 0.7f }); + m.update_state(y_true, y_pred, sample_weight: weights); + r = m.result().numpy(); + Assert.AreEqual(r, 0.6999999f); + } + /// /// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/TopKCategoricalAccuracy /// From 067c1ff92aaa35a65dc3e659111404cc8a8c052b Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 24 Feb 2023 17:35:48 -0600 Subject: [PATCH 36/52] Add metrics of F1Score and FBetaScore. --- src/TensorFlowNET.Core/APIs/tf.math.cs | 9 ++ .../Keras/Metrics/IMetricsApi.cs | 21 +++ .../Tensorflow.Binding.csproj | 6 +- src/TensorFlowNET.Keras/Metrics/F1Score.cs | 13 ++ src/TensorFlowNET.Keras/Metrics/FBetaScore.cs | 131 ++++++++++++++++++ src/TensorFlowNET.Keras/Metrics/MetricsApi.cs | 6 + .../Tensorflow.Keras.csproj | 6 +- .../Metrics/MetricsTest.cs | 28 ++++ 8 files changed, 214 insertions(+), 6 deletions(-) create mode 100644 src/TensorFlowNET.Keras/Metrics/F1Score.cs create mode 100644 src/TensorFlowNET.Keras/Metrics/FBetaScore.cs diff --git a/src/TensorFlowNET.Core/APIs/tf.math.cs b/src/TensorFlowNET.Core/APIs/tf.math.cs index 0191f8d60..dabdf126a 100644 --- a/src/TensorFlowNET.Core/APIs/tf.math.cs +++ b/src/TensorFlowNET.Core/APIs/tf.math.cs @@ -36,6 +36,15 @@ public Tensor log(Tensor x, string name = null) public Tensor erf(Tensor x, string name = null) => math_ops.erf(x, name); + public Tensor multiply(Tensor x, Tensor y, string name = null) + => math_ops.multiply(x, y, name: name); + + public Tensor divide_no_nan(Tensor a, Tensor b, string name = null) + => math_ops.div_no_nan(a, b); + + public Tensor square(Tensor x, string name = null) + => math_ops.square(x, name: name); + public Tensor sum(Tensor x, Axis? axis = null, string name = null) => math_ops.reduce_sum(x, axis: axis, name: name); diff --git a/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs b/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs index e4575620a..271ca6e12 100644 --- a/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs +++ b/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs @@ -71,6 +71,27 @@ IMetricFunc CosineSimilarity(string name = "cosine_similarity", TF_DataType dtype = TF_DataType.TF_FLOAT, Axis? axis = null); + /// + /// Computes F-1 Score. + /// + /// + IMetricFunc F1Score(int num_classes, + string? average = null, + float threshold = -1f, + string name = "fbeta_score", + TF_DataType dtype = TF_DataType.TF_FLOAT); + + /// + /// Computes F-Beta score. + /// + /// + IMetricFunc FBetaScore(int num_classes, + string? average = null, + float beta = 0.1f, + float threshold = -1f, + string name = "fbeta_score", + TF_DataType dtype = TF_DataType.TF_FLOAT); + /// /// Computes how often targets are in the top K predictions. /// diff --git a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj index c2b53e761..ecb63a7b7 100644 --- a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj +++ b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj @@ -5,7 +5,7 @@ Tensorflow.Binding Tensorflow 2.10.0 - 0.100.3 + 0.100.4 10.0 enable Haiping Chen, Meinrad Recheis, Eli Belash @@ -20,7 +20,7 @@ Google's TensorFlow full binding in .NET Standard. Building, training and infering deep learning models. https://tensorflownet.readthedocs.io - 0.100.3.0 + 0.100.4.0 tf.net 0.100.x and above are based on tensorflow native 2.10.0 @@ -38,7 +38,7 @@ https://tensorflownet.readthedocs.io tf.net 0.7x.x aligns with TensorFlow v2.7.x native library. tf.net 0.10x.x aligns with TensorFlow v2.10.x native library. - 0.100.3.0 + 0.100.4.0 LICENSE true true diff --git a/src/TensorFlowNET.Keras/Metrics/F1Score.cs b/src/TensorFlowNET.Keras/Metrics/F1Score.cs new file mode 100644 index 000000000..c3276f3e1 --- /dev/null +++ b/src/TensorFlowNET.Keras/Metrics/F1Score.cs @@ -0,0 +1,13 @@ +namespace Tensorflow.Keras.Metrics; + +public class F1Score : FBetaScore +{ + public F1Score(int num_classes, + string? average = null, + float? threshold = -1f, + string name = "f1_score", + TF_DataType dtype = TF_DataType.TF_FLOAT) + : base(num_classes, average: average, threshold: threshold, beta: 1f, name: name, dtype: dtype) + { + } +} diff --git a/src/TensorFlowNET.Keras/Metrics/FBetaScore.cs b/src/TensorFlowNET.Keras/Metrics/FBetaScore.cs new file mode 100644 index 000000000..ab4d00a96 --- /dev/null +++ b/src/TensorFlowNET.Keras/Metrics/FBetaScore.cs @@ -0,0 +1,131 @@ +namespace Tensorflow.Keras.Metrics; + +public class FBetaScore : Metric +{ + int _num_classes; + string? _average; + Tensor _beta; + Tensor _threshold; + Axis _axis; + int[] _init_shape; + + IVariableV1 true_positives; + IVariableV1 false_positives; + IVariableV1 false_negatives; + IVariableV1 weights_intermediate; + + public FBetaScore(int num_classes, + string? average = null, + float beta = 0.1f, + float? threshold = -1f, + string name = "fbeta_score", + TF_DataType dtype = TF_DataType.TF_FLOAT) + : base(name: name, dtype: dtype) + { + _num_classes = num_classes; + _average = average; + _beta = constant_op.constant(beta); + _dtype = dtype; + + if (threshold.HasValue) + { + _threshold = constant_op.constant(threshold); + } + + _init_shape = new int[0]; + + if (average != "micro") + { + _axis = 0; + _init_shape = new int[] { num_classes }; + } + + true_positives = add_weight("true_positives", shape: _init_shape, initializer: tf.initializers.zeros_initializer()); + false_positives = add_weight("false_positives", shape: _init_shape, initializer: tf.initializers.zeros_initializer()); + false_negatives = add_weight("false_negatives", shape: _init_shape, initializer: tf.initializers.zeros_initializer()); + weights_intermediate = add_weight("weights_intermediate", shape: _init_shape, initializer: tf.initializers.zeros_initializer()); + } + + public override Tensor update_state(Tensor y_true, Tensor y_pred, Tensor sample_weight = null) + { + if (_threshold == null) + { + _threshold = tf.reduce_max(y_pred, axis: -1, keepdims: true); + // make sure [0, 0, 0] doesn't become [1, 1, 1] + // Use abs(x) > eps, instead of x != 0 to check for zero + y_pred = tf.logical_and(y_pred >= _threshold, tf.abs(y_pred) > 1e-12); + } + else + { + y_pred = y_pred > _threshold; + } + + y_true = tf.cast(y_true, _dtype); + y_pred = tf.cast(y_pred, _dtype); + + true_positives.assign_add(_weighted_sum(y_pred * y_true, sample_weight)); + false_positives.assign_add( + _weighted_sum(y_pred * (1 - y_true), sample_weight) + ); + false_negatives.assign_add( + _weighted_sum((1 - y_pred) * y_true, sample_weight) + ); + weights_intermediate.assign_add(_weighted_sum(y_true, sample_weight)); + + return weights_intermediate.AsTensor(); + } + + Tensor _weighted_sum(Tensor val, Tensor? sample_weight = null) + { + if (sample_weight != null) + { + val = tf.math.multiply(val, tf.expand_dims(sample_weight, 1)); + } + + return tf.reduce_sum(val, axis: _axis); + } + + public override Tensor result() + { + var precision = tf.math.divide_no_nan( + true_positives.AsTensor(), true_positives.AsTensor() + false_positives.AsTensor() + ); + var recall = tf.math.divide_no_nan( + true_positives.AsTensor(), true_positives.AsTensor() + false_negatives.AsTensor() + ); + + var mul_value = precision * recall; + var add_value = (tf.math.square(_beta) * precision) + recall; + var mean = tf.math.divide_no_nan(mul_value, add_value); + var f1_score = mean * (1 + tf.math.square(_beta)); + + Tensor weights; + if (_average == "weighted") + { + weights = tf.math.divide_no_nan( + weights_intermediate.AsTensor(), tf.reduce_sum(weights_intermediate.AsTensor()) + ); + f1_score = tf.reduce_sum(f1_score * weights); + } + // micro, macro + else if (_average != null) + { + f1_score = tf.reduce_mean(f1_score); + } + + return f1_score; + } + + public override void reset_states() + { + var reset_value = np.zeros(_init_shape, dtype: _dtype); + keras.backend.batch_set_value( + new List<(IVariableV1, NDArray)> + { + (true_positives, reset_value), + (false_positives, reset_value), + (false_negatives, reset_value), + (weights_intermediate, reset_value) + }); + } +} diff --git a/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs b/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs index e207d27da..5230fe59a 100644 --- a/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs +++ b/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs @@ -86,6 +86,12 @@ public IMetricFunc CategoricalCrossentropy(string name = "categorical_crossentro public IMetricFunc CosineSimilarity(string name = "cosine_similarity", TF_DataType dtype = TF_DataType.TF_FLOAT, Axis? axis = null) => new CosineSimilarity(name: name, dtype: dtype, axis: axis ?? -1); + public IMetricFunc F1Score(int num_classes, string? average = null, float threshold = -1, string name = "fbeta_score", TF_DataType dtype = TF_DataType.TF_FLOAT) + => new F1Score(num_classes, average: average, threshold: threshold, name: name, dtype: dtype); + + public IMetricFunc FBetaScore(int num_classes, string? average = null, float beta = 0.1F, float threshold = -1, string name = "fbeta_score", TF_DataType dtype = TF_DataType.TF_FLOAT) + => new FBetaScore(num_classes, average: average,beta: beta, threshold: threshold, name: name, dtype: dtype); + public IMetricFunc TopKCategoricalAccuracy(int k = 5, string name = "top_k_categorical_accuracy", TF_DataType dtype = TF_DataType.TF_FLOAT) => new TopKCategoricalAccuracy(k: k, name: name, dtype: dtype); diff --git a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj index 264b9501e..104e64333 100644 --- a/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj +++ b/src/TensorFlowNET.Keras/Tensorflow.Keras.csproj @@ -7,7 +7,7 @@ enable Tensorflow.Keras AnyCPU;x64 - 0.10.3 + 0.10.4 Haiping Chen Keras for .NET Apache 2.0, Haiping Chen 2023 @@ -37,8 +37,8 @@ Keras is an API designed for human beings, not machines. Keras follows best prac Git true Open.snk - 0.10.3.0 - 0.10.3.0 + 0.10.4.0 + 0.10.4.0 LICENSE Debug;Release;GPU diff --git a/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs b/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs index 90be51bd4..2b38449b9 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs @@ -114,6 +114,34 @@ public void CosineSimilarity() Assert.AreEqual(r, 0.6999999f); } + /// + /// https://www.tensorflow.org/addons/api_docs/python/tfa/metrics/F1Score + /// + [TestMethod] + public void F1Score() + { + var y_true = np.array(new[,] { { 1, 1, 1 }, { 1, 0, 0 }, { 1, 1, 0 } }); + var y_pred = np.array(new[,] { { 0.2f, 0.6f, 0.7f }, { 0.2f, 0.6f, 0.6f }, { 0.6f, 0.8f, 0f } }); + var m = tf.keras.metrics.F1Score(num_classes: 3, threshold: 0.5f); + m.update_state(y_true, y_pred); + var r = m.result().numpy(); + Assert.AreEqual(r, new[] { 0.5f, 0.8f, 0.6666667f }); + } + + /// + /// https://www.tensorflow.org/addons/api_docs/python/tfa/metrics/FBetaScore + /// + [TestMethod] + public void FBetaScore() + { + var y_true = np.array(new[,] { { 1, 1, 1 }, { 1, 0, 0 }, { 1, 1, 0 } }); + var y_pred = np.array(new[,] { { 0.2f, 0.6f, 0.7f }, { 0.2f, 0.6f, 0.6f }, { 0.6f, 0.8f, 0f } }); + var m = tf.keras.metrics.FBetaScore(num_classes: 3, beta: 2.0f, threshold: 0.5f); + m.update_state(y_true, y_pred); + var r = m.result().numpy(); + Assert.AreEqual(r, new[] { 0.3846154f, 0.90909094f, 0.8333334f }); + } + /// /// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/TopKCategoricalAccuracy /// From 922139ff75d0ab4dd08baa8dd9f1b81f07635394 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 24 Feb 2023 18:24:19 -0600 Subject: [PATCH 37/52] Set mean of all classes in F1 Score. --- src/TensorFlowNET.Keras/Engine/Model.Train.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/TensorFlowNET.Keras/Engine/Model.Train.cs b/src/TensorFlowNET.Keras/Engine/Model.Train.cs index 0090b69e7..0151d5436 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Train.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Train.cs @@ -39,7 +39,15 @@ Dictionary train_step(DataHandler data_handler, Tensor x, Tensor compiled_metrics.update_state(y, y_pred); var dict = new Dictionary(); - metrics.ToList().ForEach(x => dict[x.Name] = (float)x.result()); + metrics.ToList().ForEach(x => + { + var r = x.result(); + if (r.ndim > 0) + { + r = tf.reduce_mean(r); + } + dict[x.Name] = (float)r; + }); return dict; } From 9726fc6f8137093c01d20a7a268839b840e004fc Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 24 Feb 2023 18:28:22 -0600 Subject: [PATCH 38/52] Fix name of f1_score. --- src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs | 2 +- src/TensorFlowNET.Keras/Metrics/MetricsApi.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs b/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs index 271ca6e12..b814a2590 100644 --- a/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs +++ b/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs @@ -78,7 +78,7 @@ IMetricFunc CosineSimilarity(string name = "cosine_similarity", IMetricFunc F1Score(int num_classes, string? average = null, float threshold = -1f, - string name = "fbeta_score", + string name = "f1_score", TF_DataType dtype = TF_DataType.TF_FLOAT); /// diff --git a/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs b/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs index 5230fe59a..b9fbe180f 100644 --- a/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs +++ b/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs @@ -86,7 +86,7 @@ public IMetricFunc CategoricalCrossentropy(string name = "categorical_crossentro public IMetricFunc CosineSimilarity(string name = "cosine_similarity", TF_DataType dtype = TF_DataType.TF_FLOAT, Axis? axis = null) => new CosineSimilarity(name: name, dtype: dtype, axis: axis ?? -1); - public IMetricFunc F1Score(int num_classes, string? average = null, float threshold = -1, string name = "fbeta_score", TF_DataType dtype = TF_DataType.TF_FLOAT) + public IMetricFunc F1Score(int num_classes, string? average = null, float threshold = -1, string name = "f1_score", TF_DataType dtype = TF_DataType.TF_FLOAT) => new F1Score(num_classes, average: average, threshold: threshold, name: name, dtype: dtype); public IMetricFunc FBetaScore(int num_classes, string? average = null, float beta = 0.1F, float threshold = -1, string name = "fbeta_score", TF_DataType dtype = TF_DataType.TF_FLOAT) From 32a3e48d197ae879d24ced4050d932cf434b1b9d Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 24 Feb 2023 18:42:55 -0600 Subject: [PATCH 39/52] All threshold as null in F1 Score. --- src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs | 4 ++-- src/TensorFlowNET.Keras/Metrics/F1Score.cs | 2 +- src/TensorFlowNET.Keras/Metrics/FBetaScore.cs | 2 +- src/TensorFlowNET.Keras/Metrics/MetricsApi.cs | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs b/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs index b814a2590..64c2c14fe 100644 --- a/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs +++ b/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs @@ -77,7 +77,7 @@ IMetricFunc CosineSimilarity(string name = "cosine_similarity", /// IMetricFunc F1Score(int num_classes, string? average = null, - float threshold = -1f, + float? threshold = null, string name = "f1_score", TF_DataType dtype = TF_DataType.TF_FLOAT); @@ -88,7 +88,7 @@ IMetricFunc F1Score(int num_classes, IMetricFunc FBetaScore(int num_classes, string? average = null, float beta = 0.1f, - float threshold = -1f, + float? threshold = null, string name = "fbeta_score", TF_DataType dtype = TF_DataType.TF_FLOAT); diff --git a/src/TensorFlowNET.Keras/Metrics/F1Score.cs b/src/TensorFlowNET.Keras/Metrics/F1Score.cs index c3276f3e1..fc24136d8 100644 --- a/src/TensorFlowNET.Keras/Metrics/F1Score.cs +++ b/src/TensorFlowNET.Keras/Metrics/F1Score.cs @@ -4,7 +4,7 @@ public class F1Score : FBetaScore { public F1Score(int num_classes, string? average = null, - float? threshold = -1f, + float? threshold = null, string name = "f1_score", TF_DataType dtype = TF_DataType.TF_FLOAT) : base(num_classes, average: average, threshold: threshold, beta: 1f, name: name, dtype: dtype) diff --git a/src/TensorFlowNET.Keras/Metrics/FBetaScore.cs b/src/TensorFlowNET.Keras/Metrics/FBetaScore.cs index ab4d00a96..39e3e9af9 100644 --- a/src/TensorFlowNET.Keras/Metrics/FBetaScore.cs +++ b/src/TensorFlowNET.Keras/Metrics/FBetaScore.cs @@ -17,7 +17,7 @@ public class FBetaScore : Metric public FBetaScore(int num_classes, string? average = null, float beta = 0.1f, - float? threshold = -1f, + float? threshold = null, string name = "fbeta_score", TF_DataType dtype = TF_DataType.TF_FLOAT) : base(name: name, dtype: dtype) diff --git a/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs b/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs index b9fbe180f..bd12f82ae 100644 --- a/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs +++ b/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs @@ -86,10 +86,10 @@ public IMetricFunc CategoricalCrossentropy(string name = "categorical_crossentro public IMetricFunc CosineSimilarity(string name = "cosine_similarity", TF_DataType dtype = TF_DataType.TF_FLOAT, Axis? axis = null) => new CosineSimilarity(name: name, dtype: dtype, axis: axis ?? -1); - public IMetricFunc F1Score(int num_classes, string? average = null, float threshold = -1, string name = "f1_score", TF_DataType dtype = TF_DataType.TF_FLOAT) + public IMetricFunc F1Score(int num_classes, string? average = null, float? threshold = null, string name = "f1_score", TF_DataType dtype = TF_DataType.TF_FLOAT) => new F1Score(num_classes, average: average, threshold: threshold, name: name, dtype: dtype); - public IMetricFunc FBetaScore(int num_classes, string? average = null, float beta = 0.1F, float threshold = -1, string name = "fbeta_score", TF_DataType dtype = TF_DataType.TF_FLOAT) + public IMetricFunc FBetaScore(int num_classes, string? average = null, float beta = 0.1F, float? threshold = null, string name = "fbeta_score", TF_DataType dtype = TF_DataType.TF_FLOAT) => new FBetaScore(num_classes, average: average,beta: beta, threshold: threshold, name: name, dtype: dtype); public IMetricFunc TopKCategoricalAccuracy(int k = 5, string name = "top_k_categorical_accuracy", TF_DataType dtype = TF_DataType.TF_FLOAT) From c72f47990e704e889b965cf95c4a5b690c757641 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sat, 25 Feb 2023 11:26:24 -0600 Subject: [PATCH 40/52] Add metric of HammingLoss. --- src/TensorFlowNET.Core/APIs/tf.math.cs | 2 + .../Keras/Metrics/IMetricsApi.cs | 15 ++++++- src/TensorFlowNET.Core/Operations/math_ops.cs | 12 +++++ .../Tensorflow.Binding.csproj | 2 +- .../Metrics/HammingLoss.cs | 15 +++++++ src/TensorFlowNET.Keras/Metrics/MetricsApi.cs | 3 ++ .../Metrics/metrics_utils.cs | 30 +++++++++++++ .../Metrics/MetricsTest.cs | 45 +++++++++++++++++++ 8 files changed, 122 insertions(+), 2 deletions(-) create mode 100644 src/TensorFlowNET.Keras/Metrics/HammingLoss.cs diff --git a/src/TensorFlowNET.Core/APIs/tf.math.cs b/src/TensorFlowNET.Core/APIs/tf.math.cs index dabdf126a..61b7caf48 100644 --- a/src/TensorFlowNET.Core/APIs/tf.math.cs +++ b/src/TensorFlowNET.Core/APIs/tf.math.cs @@ -24,6 +24,8 @@ public class MathApi public Tensor argmax(Tensor input, Axis axis = null, string name = null, int? dimension = null, TF_DataType output_type = TF_DataType.TF_INT64) => gen_math_ops.arg_max(input, axis, name: name, output_type: output_type); + public Tensor count_nonzero(Tensor input, Axis? axis = null, bool? keepdims = null, TF_DataType dtype = TF_DataType.TF_INT64, string name = null) + => math_ops.count_nonzero_v2(input, axis: axis, keepdims: keepdims ?? false, dtype: dtype); public Tensor log(Tensor x, string name = null) => gen_math_ops.log(x, name); diff --git a/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs b/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs index 64c2c14fe..5d08cc78e 100644 --- a/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs +++ b/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs @@ -91,7 +91,20 @@ IMetricFunc FBetaScore(int num_classes, float? threshold = null, string name = "fbeta_score", TF_DataType dtype = TF_DataType.TF_FLOAT); - + + /// + /// Computes hamming loss. + /// + /// multiclass or multilabel + /// + /// + /// + /// + IMetricFunc HammingLoss(string mode, + float? threshold = null, + string name = "hamming_loss", + TF_DataType dtype = TF_DataType.TF_FLOAT); + /// /// Computes how often targets are in the top K predictions. /// diff --git a/src/TensorFlowNET.Core/Operations/math_ops.cs b/src/TensorFlowNET.Core/Operations/math_ops.cs index 9542f6436..36f7db794 100644 --- a/src/TensorFlowNET.Core/Operations/math_ops.cs +++ b/src/TensorFlowNET.Core/Operations/math_ops.cs @@ -821,6 +821,18 @@ public static Tensor batch_matmul(Tensor x, Tensor y, .SetAttributes(new { adj_x, adj_y })); }); + public static Tensor count_nonzero_v2(Tensor input, + Axis? axis, + bool keepdims = false, + string name = null, + TF_DataType dtype = TF_DataType.TF_INT64) + => tf_with(ops.name_scope(name, "count_nonzero", input), scope => + { + name = scope; + var zero = array_ops.zeros(Shape.Scalar, dtype: input.dtype); + return reduce_sum(cast(gen_math_ops.not_equal(input, zero), dtype), axis: axis, keepdims: keepdims); + }); + public static Tensor bincount(Tensor arr, Tensor weights = null, Tensor minlength = null, Tensor maxlength = null, diff --git a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj index ecb63a7b7..8925228dc 100644 --- a/src/TensorFlowNET.Core/Tensorflow.Binding.csproj +++ b/src/TensorFlowNET.Core/Tensorflow.Binding.csproj @@ -109,7 +109,7 @@ https://tensorflownet.readthedocs.io - + diff --git a/src/TensorFlowNET.Keras/Metrics/HammingLoss.cs b/src/TensorFlowNET.Keras/Metrics/HammingLoss.cs new file mode 100644 index 000000000..2b65424e9 --- /dev/null +++ b/src/TensorFlowNET.Keras/Metrics/HammingLoss.cs @@ -0,0 +1,15 @@ +namespace Tensorflow.Keras.Metrics; + +public class HammingLoss : MeanMetricWrapper +{ + public HammingLoss(string mode, + NDArray threshold = null, + string name = "hamming_loss", + TF_DataType dtype = TF_DataType.TF_FLOAT) + : base((yt, yp) => metrics_utils.hamming_loss_fn(yt, yp, threshold, mode), + name: name, + dtype: dtype) + { + _dtype = dtype; + } +} diff --git a/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs b/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs index bd12f82ae..585fefae2 100644 --- a/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs +++ b/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs @@ -92,6 +92,9 @@ public IMetricFunc F1Score(int num_classes, string? average = null, float? thres public IMetricFunc FBetaScore(int num_classes, string? average = null, float beta = 0.1F, float? threshold = null, string name = "fbeta_score", TF_DataType dtype = TF_DataType.TF_FLOAT) => new FBetaScore(num_classes, average: average,beta: beta, threshold: threshold, name: name, dtype: dtype); + public IMetricFunc HammingLoss(string mode, float? threshold = null, string name = "hamming_loss", TF_DataType dtype = TF_DataType.TF_FLOAT) + => new HammingLoss(mode, threshold: threshold, name: name, dtype: dtype); + public IMetricFunc TopKCategoricalAccuracy(int k = 5, string name = "top_k_categorical_accuracy", TF_DataType dtype = TF_DataType.TF_FLOAT) => new TopKCategoricalAccuracy(k: k, name: name, dtype: dtype); diff --git a/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs b/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs index f4bfc3da4..69cc789e9 100644 --- a/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs +++ b/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs @@ -24,6 +24,36 @@ public static Tensor cosine_similarity(Tensor y_true, Tensor y_pred, Axis? axis return tf.reduce_sum(y_true * y_pred, axis: axis ?? -1); } + public static Tensor hamming_loss_fn(Tensor y_true, Tensor y_pred, Tensor threshold, string mode) + { + if (threshold == null) + { + threshold = tf.reduce_max(y_pred, axis: -1, keepdims: true); + // make sure [0, 0, 0] doesn't become [1, 1, 1] + // Use abs(x) > eps, instead of x != 0 to check for zero + y_pred = tf.logical_and(y_pred >= threshold, tf.abs(y_pred) > 1e-12); + } + else + { + y_pred = y_pred > threshold; + } + + + y_true = tf.cast(y_true, tf.int32); + y_pred = tf.cast(y_pred, tf.int32); + + if (mode == "multiclass") + { + var nonzero = tf.cast(tf.math.count_nonzero(y_true * y_pred, axis: -1), tf.float32); + return 1.0 - nonzero; + } + else + { + var nonzero = tf.cast(tf.math.count_nonzero(y_true - y_pred, axis: -1), tf.float32); + return nonzero / y_true.shape[-1]; + } + } + /// /// Creates float Tensor, 1.0 for label-prediction match, 0.0 for mismatch. /// diff --git a/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs b/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs index 2b38449b9..267cef815 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs @@ -142,6 +142,51 @@ public void FBetaScore() Assert.AreEqual(r, new[] { 0.3846154f, 0.90909094f, 0.8333334f }); } + /// + /// https://www.tensorflow.org/addons/api_docs/python/tfa/metrics/HammingLoss + /// + [TestMethod] + public void HammingLoss() + { + // multi-class hamming loss + var y_true = np.array(new[,] + { + { 1, 0, 0, 0 }, + { 0, 0, 1, 0 }, + { 0, 0, 0, 1 }, + { 0, 1, 0, 0 } + }); + var y_pred = np.array(new[,] + { + { 0.8f, 0.1f, 0.1f, 0.0f }, + { 0.2f, 0.0f, 0.8f, 0.0f }, + { 0.05f, 0.05f, 0.1f, 0.8f }, + { 1.0f, 0.0f, 0.0f, 0.0f } + }); + var m = tf.keras.metrics.HammingLoss(mode: "multiclass", threshold: 0.6f); + m.update_state(y_true, y_pred); + var r = m.result().numpy(); + Assert.AreEqual(r, 0.25f); + + // multi-label hamming loss + y_true = np.array(new[,] + { + { 1, 0, 1, 0 }, + { 0, 1, 0, 1 }, + { 0, 0, 0, 1 } + }); + y_pred = np.array(new[,] + { + { 0.82f, 0.5f, 0.9f, 0.0f }, + { 0f, 1f, 0.4f, 0.98f }, + { 0.89f, 0.79f, 0f, 0.3f } + }); + m = tf.keras.metrics.HammingLoss(mode: "multilabel", threshold: 0.8f); + m.update_state(y_true, y_pred); + r = m.result().numpy(); + Assert.AreEqual(r, 0.16666667f); + } + /// /// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/TopKCategoricalAccuracy /// From d7945764867e19f21dffc06b7a1ce82b8ec5017c Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sat, 25 Feb 2023 12:11:34 -0600 Subject: [PATCH 41/52] Add activations.mish. --- src/TensorFlowNET.Core/APIs/tf.math.cs | 8 ++++++++ src/TensorFlowNET.Core/Gradients/nn_grad.cs | 10 ++++++++++ src/TensorFlowNET.Core/Operations/nn_ops.cs | 3 +++ src/TensorFlowNET.Keras/Activations.cs | 6 +++++- .../Layers/ActivationTest.cs | 11 +++++++++++ 5 files changed, 37 insertions(+), 1 deletion(-) diff --git a/src/TensorFlowNET.Core/APIs/tf.math.cs b/src/TensorFlowNET.Core/APIs/tf.math.cs index 61b7caf48..c7aa46704 100644 --- a/src/TensorFlowNET.Core/APIs/tf.math.cs +++ b/src/TensorFlowNET.Core/APIs/tf.math.cs @@ -14,6 +14,8 @@ You may obtain a copy of the License at limitations under the License. ******************************************************************************/ +using Tensorflow.Operations; + namespace Tensorflow { public partial class tensorflow @@ -50,6 +52,12 @@ public Tensor square(Tensor x, string name = null) public Tensor sum(Tensor x, Axis? axis = null, string name = null) => math_ops.reduce_sum(x, axis: axis, name: name); + public Tensor softplus(Tensor features, string name = null) + => nn_ops.softplus(features, name: name); + + public Tensor tanh(Tensor x, string name = null) + => math_ops.tanh(x, name: name); + /// /// Finds values and indices of the `k` largest entries for the last dimension. /// diff --git a/src/TensorFlowNET.Core/Gradients/nn_grad.cs b/src/TensorFlowNET.Core/Gradients/nn_grad.cs index d461595b1..15b72f55c 100644 --- a/src/TensorFlowNET.Core/Gradients/nn_grad.cs +++ b/src/TensorFlowNET.Core/Gradients/nn_grad.cs @@ -120,6 +120,16 @@ public static Tensor[] _SparseSoftmaxCrossEntropyWithLogitsGrad(Operation op, Te }; } + [RegisterGradient("Softplus")] + public static Tensor[] _SoftplusGrad(Operation op, Tensor[] grads) + { + var grad = grads[0]; + var x = op.inputs[0]; + + var softplus = grad * math_ops.sigmoid(x); + return new Tensor[] { softplus }; + } + [RegisterGradient("SquaredDifference")] public static Tensor[] _SquaredDifferenceGrad(Operation op, Tensor[] grads) { diff --git a/src/TensorFlowNET.Core/Operations/nn_ops.cs b/src/TensorFlowNET.Core/Operations/nn_ops.cs index 5877d234d..b8d5103c4 100644 --- a/src/TensorFlowNET.Core/Operations/nn_ops.cs +++ b/src/TensorFlowNET.Core/Operations/nn_ops.cs @@ -132,6 +132,9 @@ public static Tensor softmax(Tensor logits, int axis = -1, string name = null) return _softmax(logits, gen_nn_ops.softmax, axis, name); } + public static Tensor softplus(Tensor features, string name = null) + => tf.Context.ExecuteOp("Softplus", name, new ExecuteOpArgs(features)); + public static Tensor l2_loss(Tensor t, string name = null) => tf.Context.ExecuteOp("L2Loss", name, new ExecuteOpArgs(t)); diff --git a/src/TensorFlowNET.Keras/Activations.cs b/src/TensorFlowNET.Keras/Activations.cs index 444c783e0..37bddac76 100644 --- a/src/TensorFlowNET.Keras/Activations.cs +++ b/src/TensorFlowNET.Keras/Activations.cs @@ -20,12 +20,14 @@ public class Activations => tf.Context.ExecuteOp("Softmax", name, new ExecuteOpArgs(features)); private static Activation _tanh = (features, name) => tf.Context.ExecuteOp("Tanh", name, new ExecuteOpArgs(features)); + private static Activation _mish = (features, name) + => features * tf.math.tanh(tf.math.softplus(features)); /// /// Register the name-activation mapping in this static class. /// /// - /// + /// private static void RegisterActivation(string name, Activation activation) { _nameActivationMap[name] = activation; @@ -42,6 +44,7 @@ static Activations() RegisterActivation("sigmoid", _sigmoid); RegisterActivation("softmax", _softmax); RegisterActivation("tanh", _tanh); + RegisterActivation("mish", _mish); } public Activation Linear => _linear; @@ -54,6 +57,7 @@ static Activations() public Activation Tanh => _tanh; + public Activation Mish => _mish; public static Activation GetActivationByName(string name) { diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/ActivationTest.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/ActivationTest.cs index 904601b35..1f45c518f 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/ActivationTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/ActivationTest.cs @@ -94,5 +94,16 @@ public void Swish() NDArray expected = new NDArray(new float[] { -0.14227762f, -0.23840584f, -0.26894143f, 0f, 0.7310586f, 1.761594f }); Assert.AreEqual(expected, output.numpy()); } + + /// + /// https://www.tensorflow.org/addons/api_docs/python/tfa/activations/mish + /// + [TestMethod] + public void Mish() + { + var x = tf.constant(new[] { 1.0, 0.0, 1.0 }, dtype: tf.float32); + var output = keras.activations.Mish(x); + Assert.AreEqual(new[] { 0.86509836f, 0f, 0.86509836f }, output.numpy()); + } } } From 55cc4d0b78fd9f0fd2d30a20861eeba73a4ddad0 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sun, 26 Feb 2023 10:23:33 -0600 Subject: [PATCH 42/52] Support save and load in numpy.lib.format. --- src/TensorFlowNET.Core/GlobalUsing.cs | 3 + .../NumPy/Implementation/NumPyImpl.load.cs | 19 +- .../NumPy/Numpy.Persistence.cs | 60 +++++ .../NumPy/Persistence/NpyFormat.cs | 95 ++++++++ .../NumPy/Persistence/NpzDictionary.cs | 180 +++++++++++++++ .../NumPy/Persistence/NpzDictionaryArray.cs | 138 ++++++++++++ .../NumPy/Persistence/NpzFormat.cs | 37 ++++ src/TensorFlowNET.Core/Numpy/NpzDictionary.cs | 206 ------------------ .../Numpy/Numpy.Creation.cs | 8 +- src/TensorFlowNET.Core/Numpy/Numpy.cs | 101 ++++----- .../NumPy/Persistence.Test.cs | 42 ++++ 11 files changed, 608 insertions(+), 281 deletions(-) create mode 100644 src/TensorFlowNET.Core/NumPy/Numpy.Persistence.cs create mode 100644 src/TensorFlowNET.Core/NumPy/Persistence/NpyFormat.cs create mode 100644 src/TensorFlowNET.Core/NumPy/Persistence/NpzDictionary.cs create mode 100644 src/TensorFlowNET.Core/NumPy/Persistence/NpzDictionaryArray.cs create mode 100644 src/TensorFlowNET.Core/NumPy/Persistence/NpzFormat.cs delete mode 100644 src/TensorFlowNET.Core/Numpy/NpzDictionary.cs create mode 100644 test/TensorFlowNET.UnitTest/NumPy/Persistence.Test.cs diff --git a/src/TensorFlowNET.Core/GlobalUsing.cs b/src/TensorFlowNET.Core/GlobalUsing.cs index fe77202ce..2fd5b437b 100644 --- a/src/TensorFlowNET.Core/GlobalUsing.cs +++ b/src/TensorFlowNET.Core/GlobalUsing.cs @@ -1,3 +1,6 @@ global using System; global using System.Collections.Generic; global using System.Text; +global using System.Collections; +global using System.Data; +global using System.Linq; \ No newline at end of file diff --git a/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.load.cs b/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.load.cs index 70a4245b3..05f53d5e7 100644 --- a/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.load.cs +++ b/src/TensorFlowNET.Core/NumPy/Implementation/NumPyImpl.load.cs @@ -1,11 +1,4 @@ - using System; -using System.Collections; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Reflection; -using System.Text; -using Tensorflow.Util; +using System.IO; namespace Tensorflow.NumPy { @@ -15,10 +8,7 @@ public NDArray load(string file) { using var stream = new FileStream(file, FileMode.Open); using var reader = new BinaryReader(stream, Encoding.ASCII, leaveOpen: true); - int bytes; - Type type; - int[] shape; - if (!ParseReader(reader, out bytes, out type, out shape)) + if (!ParseReader(reader, out var bytes, out var type, out var shape)) throw new FormatException(); Array array = Create(type, shape.Aggregate((dims, dim) => dims * dim)); @@ -31,10 +21,7 @@ public Array LoadMatrix(Stream stream) { using (var reader = new BinaryReader(stream, System.Text.Encoding.ASCII, leaveOpen: true)) { - int bytes; - Type type; - int[] shape; - if (!ParseReader(reader, out bytes, out type, out shape)) + if (!ParseReader(reader, out var bytes, out var type, out var shape)) throw new FormatException(); Array matrix = Array.CreateInstance(type, shape); diff --git a/src/TensorFlowNET.Core/NumPy/Numpy.Persistence.cs b/src/TensorFlowNET.Core/NumPy/Numpy.Persistence.cs new file mode 100644 index 000000000..b349f5229 --- /dev/null +++ b/src/TensorFlowNET.Core/NumPy/Numpy.Persistence.cs @@ -0,0 +1,60 @@ +/***************************************************************************** + Copyright 2023 Haiping Chen. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +******************************************************************************/ + +using System.IO; +using System.IO.Compression; + +namespace Tensorflow.NumPy; + +public partial class np +{ + [AutoNumPy] + public static NpzDictionary loadz(string file) + { + using var stream = new FileStream(file, FileMode.Open); + return new NpzDictionary(stream); + } + + public static void save(string file, NDArray nd) + { + using var stream = new FileStream(file, FileMode.Create); + NpyFormat.Save(nd, stream); + } + + public static void savez(string file, params NDArray[] nds) + { + using var stream = new FileStream(file, FileMode.Create); + NpzFormat.Save(nds, stream); + } + + public static void savez(string file, object nds) + { + using var stream = new FileStream(file, FileMode.Create); + NpzFormat.Save(nds, stream); + } + + public static void savez_compressed(string file, params NDArray[] nds) + { + using var stream = new FileStream(file, FileMode.Create); + NpzFormat.Save(nds, stream, CompressionLevel.Fastest); + } + + public static void savez_compressed(string file, object nds) + { + using var stream = new FileStream(file, FileMode.Create); + NpzFormat.Save(nds, stream, CompressionLevel.Fastest); + } +} diff --git a/src/TensorFlowNET.Core/NumPy/Persistence/NpyFormat.cs b/src/TensorFlowNET.Core/NumPy/Persistence/NpyFormat.cs new file mode 100644 index 000000000..1886e4b4e --- /dev/null +++ b/src/TensorFlowNET.Core/NumPy/Persistence/NpyFormat.cs @@ -0,0 +1,95 @@ +using System.IO; +using System.Runtime.InteropServices; + +namespace Tensorflow.NumPy; + +public class NpyFormat +{ + public static void Save(NDArray array, Stream stream, bool leaveOpen = true) + { + using var writer = new BinaryWriter(stream, Encoding.ASCII, leaveOpen: leaveOpen); + + string dtype = GetDtypeName(array, out var type, out var maxLength); + int[] shape = array.shape.as_int_list(); + var bytesWritten = (ulong)writeHeader(writer, dtype, shape); + stream.Write(array.ToByteArray(), 0, (int)array.bytesize); + } + + private static int writeHeader(BinaryWriter writer, string dtype, int[] shape) + { + // The first 6 bytes are a magic string: exactly "x93NUMPY" + + char[] magic = { 'N', 'U', 'M', 'P', 'Y' }; + writer.Write((byte)147); + writer.Write(magic); + writer.Write((byte)1); // major + writer.Write((byte)0); // minor; + + string tuple = shape.Length == 1 ? $"{shape[0]}," : String.Join(", ", shape.Select(i => i.ToString()).ToArray()); + string header = "{{'descr': '{0}', 'fortran_order': False, 'shape': ({1}), }}"; + header = string.Format(header, dtype, tuple); + int preamble = 10; // magic string (6) + 4 + + int len = header.Length + 1; // the 1 is to account for the missing \n at the end + int headerSize = len + preamble; + + int pad = 16 - (headerSize % 16); + header = header.PadRight(header.Length + pad); + header += "\n"; + headerSize = header.Length + preamble; + + if (headerSize % 16 != 0) + throw new Exception(""); + + writer.Write((ushort)header.Length); + for (int i = 0; i < header.Length; i++) + writer.Write((byte)header[i]); + + return headerSize; + } + + private static string GetDtypeName(NDArray array, out Type type, out int bytes) + { + type = array.dtype.as_system_dtype(); + + bytes = 1; + + if (type == typeof(string)) + { + throw new NotSupportedException(""); + } + else if (type == typeof(bool)) + { + bytes = 1; + } + else + { + bytes = Marshal.SizeOf(type); + } + + if (type == typeof(bool)) + return "|b1"; + else if (type == typeof(byte)) + return "|i1"; + else if (type == typeof(short)) + return " : IDisposable, IReadOnlyDictionary, ICollection + where T : class, + ICloneable, IList, ICollection, IEnumerable, IStructuralComparable, IStructuralEquatable +{ + Stream stream; + ZipArchive archive; + + bool disposedValue = false; + + Dictionary entries; + Dictionary arrays; + + + public NpzDictionary(Stream stream) + { + this.stream = stream; + this.archive = new ZipArchive(stream, ZipArchiveMode.Read, leaveOpen: true); + + this.entries = new Dictionary(); + foreach (var entry in archive.Entries) + this.entries[entry.FullName] = entry; + + this.arrays = new Dictionary(); + } + + + public IEnumerable Keys + { + get { return entries.Keys; } + } + + + public IEnumerable Values + { + get { return entries.Values.Select(OpenEntry); } + } + + public int Count + { + get { return entries.Count; } + } + + + public object SyncRoot + { + get { return ((ICollection)entries).SyncRoot; } + } + + + public bool IsSynchronized + { + get { return ((ICollection)entries).IsSynchronized; } + } + + public bool IsReadOnly + { + get { return true; } + } + + public T this[string key] + { + get { return OpenEntry(entries[key]); } + } + + private T OpenEntry(ZipArchiveEntry entry) + { + T array; + if (arrays.TryGetValue(entry.FullName, out array)) + return array; + + using (Stream s = entry.Open()) + { + array = Load_Npz(s); + arrays[entry.FullName] = array; + return array; + } + } + + protected virtual T Load_Npz(Stream s) + { + return np.Load(s); + } + + public bool ContainsKey(string key) + { + return entries.ContainsKey(key); + } + + public bool TryGetValue(string key, out T value) + { + value = default(T); + ZipArchiveEntry entry; + if (!entries.TryGetValue(key, out entry)) + return false; + value = OpenEntry(entry); + return true; + } + + public IEnumerator> GetEnumerator() + { + foreach (var entry in archive.Entries) + yield return new KeyValuePair(entry.FullName, OpenEntry(entry)); + } + + IEnumerator IEnumerable.GetEnumerator() + { + foreach (var entry in archive.Entries) + yield return new KeyValuePair(entry.FullName, OpenEntry(entry)); + } + + IEnumerator IEnumerable.GetEnumerator() + { + foreach (var entry in archive.Entries) + yield return OpenEntry(entry); + } + + public void CopyTo(Array array, int arrayIndex) + { + foreach (var v in this) + array.SetValue(v, arrayIndex++); + } + + public void CopyTo(T[] array, int arrayIndex) + { + foreach (var v in this) + array.SetValue(v, arrayIndex++); + } + + public void Add(T item) + { + throw new ReadOnlyException(); + } + + public void Clear() + { + throw new ReadOnlyException(); + } + + public bool Contains(T item) + { + foreach (var v in this) + if (Object.Equals(v.Value, item)) + return true; + return false; + } + + public bool Remove(T item) + { + throw new ReadOnlyException(); + } + + protected virtual void Dispose(bool disposing) + { + if (!disposedValue) + { + if (disposing) + { + archive.Dispose(); + stream.Dispose(); + } + + archive = null; + stream = null; + entries = null; + arrays = null; + + disposedValue = true; + } + } + + public void Dispose() + { + Dispose(true); + } +} diff --git a/src/TensorFlowNET.Core/NumPy/Persistence/NpzDictionaryArray.cs b/src/TensorFlowNET.Core/NumPy/Persistence/NpzDictionaryArray.cs new file mode 100644 index 000000000..6e81216ea --- /dev/null +++ b/src/TensorFlowNET.Core/NumPy/Persistence/NpzDictionaryArray.cs @@ -0,0 +1,138 @@ +using System.IO; +using System.IO.Compression; + +namespace Tensorflow.NumPy; + +public class NpzDictionary +{ + Dictionary arrays = new Dictionary(); + + public NDArray this[string key] => arrays[key]; + + public NpzDictionary(Stream stream) + { + using var archive = new ZipArchive(stream, ZipArchiveMode.Read, leaveOpen: false); + + foreach (var entry in archive.Entries) + { + arrays[entry.FullName] = OpenEntry(entry); + } + } + + private NDArray OpenEntry(ZipArchiveEntry entry) + { + if (arrays.TryGetValue(entry.FullName, out var array)) + return array; + + using var s = entry.Open(); + return LoadMatrix(s); + } + + public Array LoadMatrix(Stream stream) + { + using var reader = new BinaryReader(stream, System.Text.Encoding.ASCII, leaveOpen: false); + + if (!ParseReader(reader, out var bytes, out var type, out var shape)) + throw new FormatException(); + + Array matrix = Array.CreateInstance(type, shape); + + return ReadMatrix(reader, matrix, bytes, type, shape); + } + + bool ParseReader(BinaryReader reader, out int bytes, out Type t, out int[] shape) + { + bytes = 0; + t = null; + shape = null; + + // The first 6 bytes are a magic string: exactly "x93NUMPY" + if (reader.ReadChar() != 63) return false; + if (reader.ReadChar() != 'N') return false; + if (reader.ReadChar() != 'U') return false; + if (reader.ReadChar() != 'M') return false; + if (reader.ReadChar() != 'P') return false; + if (reader.ReadChar() != 'Y') return false; + + byte major = reader.ReadByte(); // 1 + byte minor = reader.ReadByte(); // 0 + + if (major != 1 || minor != 0) + throw new NotSupportedException(); + + ushort len = reader.ReadUInt16(); + + string header = new string(reader.ReadChars(len)); + string mark = "'descr': '"; + int s = header.IndexOf(mark) + mark.Length; + int e = header.IndexOf("'", s + 1); + string type = header.Substring(s, e - s); + bool? isLittleEndian; + t = GetType(type, out bytes, out isLittleEndian); + + if (isLittleEndian.HasValue && isLittleEndian.Value == false) + throw new Exception(); + + mark = "'fortran_order': "; + s = header.IndexOf(mark) + mark.Length; + e = header.IndexOf(",", s + 1); + bool fortran = bool.Parse(header.Substring(s, e - s)); + + if (fortran) + throw new Exception(); + + mark = "'shape': ("; + s = header.IndexOf(mark) + mark.Length; + e = header.IndexOf(")", s + 1); + shape = header.Substring(s, e - s).Split(',').Where(v => !String.IsNullOrEmpty(v)).Select(Int32.Parse).ToArray(); + + return true; + } + + Type GetType(string dtype, out int bytes, out bool? isLittleEndian) + { + isLittleEndian = IsLittleEndian(dtype); + bytes = int.Parse(dtype.Substring(2)); + + string typeCode = dtype.Substring(1); + return typeCode switch + { + "b1" => typeof(bool), + "i1" => typeof(byte), + "i2" => typeof(short), + "i4" => typeof(int), + "i8" => typeof(long), + "u1" => typeof(byte), + "u2" => typeof(ushort), + "u4" => typeof(uint), + "u8" => typeof(ulong), + "f4" => typeof(float), + "f8" => typeof(double), + // typeCode.StartsWith("S") => typeof(string), + _ => throw new NotSupportedException() + }; + } + + bool? IsLittleEndian(string type) + { + return type[0] switch + { + '<' => true, + '>' => false, + '|' => null, + _ => throw new Exception() + }; + } + + Array ReadMatrix(BinaryReader reader, Array matrix, int bytes, Type type, int[] shape) + { + int total = 1; + for (int i = 0; i < shape.Length; i++) + total *= shape[i]; + + var buffer = reader.ReadBytes(bytes * total); + System.Buffer.BlockCopy(buffer, 0, matrix, 0, buffer.Length); + + return matrix; + } +} diff --git a/src/TensorFlowNET.Core/NumPy/Persistence/NpzFormat.cs b/src/TensorFlowNET.Core/NumPy/Persistence/NpzFormat.cs new file mode 100644 index 000000000..7470a1ea7 --- /dev/null +++ b/src/TensorFlowNET.Core/NumPy/Persistence/NpzFormat.cs @@ -0,0 +1,37 @@ +using System.IO.Compression; +using System.IO; +using System; + +namespace Tensorflow.NumPy; + +public class NpzFormat +{ + public static void Save(NDArray[] arrays, Stream stream, CompressionLevel compression = CompressionLevel.NoCompression, bool leaveOpen = false) + { + using var zip = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: leaveOpen); + for (int i = 0; i < arrays.Length; i++) + { + var entry = zip.CreateEntry($"arr_{i}", compression); + NpyFormat.Save(arrays[i], entry.Open(), leaveOpen); + } + } + + public static void Save(object arrays, Stream stream, CompressionLevel compression = CompressionLevel.NoCompression, bool leaveOpen = false) + { + var properties = arrays.GetType().GetProperties(); + using var zip = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: leaveOpen); + for (int i = 0; i < properties.Length; i++) + { + var entry = zip.CreateEntry(properties[i].Name, compression); + var value = properties[i].GetValue(arrays); + if (value is NDArray nd) + { + NpyFormat.Save(nd, entry.Open(), leaveOpen); + } + else + { + throw new NotSupportedException("Please pass in NDArray."); + } + } + } +} diff --git a/src/TensorFlowNET.Core/Numpy/NpzDictionary.cs b/src/TensorFlowNET.Core/Numpy/NpzDictionary.cs deleted file mode 100644 index bb7ff693e..000000000 --- a/src/TensorFlowNET.Core/Numpy/NpzDictionary.cs +++ /dev/null @@ -1,206 +0,0 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.Data; -using System.IO; -using System.IO.Compression; -using System.Linq; -using System.Text; -using Tensorflow.Util; - -namespace Tensorflow.NumPy -{ - public class NpzDictionary : IDisposable, IReadOnlyDictionary, ICollection - where T : class, - ICloneable, IList, ICollection, IEnumerable, IStructuralComparable, IStructuralEquatable - { - Stream stream; - ZipArchive archive; - - bool disposedValue = false; - - Dictionary entries; - Dictionary arrays; - - - public NpzDictionary(Stream stream) - { - this.stream = stream; - this.archive = new ZipArchive(stream, ZipArchiveMode.Read, leaveOpen: true); - - this.entries = new Dictionary(); - foreach (var entry in archive.Entries) - this.entries[entry.FullName] = entry; - - this.arrays = new Dictionary(); - } - - - public IEnumerable Keys - { - get { return entries.Keys; } - } - - - public IEnumerable Values - { - get { return entries.Values.Select(OpenEntry); } - } - - public int Count - { - get { return entries.Count; } - } - - - public object SyncRoot - { - get { return ((ICollection)entries).SyncRoot; } - } - - - public bool IsSynchronized - { - get { return ((ICollection)entries).IsSynchronized; } - } - - public bool IsReadOnly - { - get { return true; } - } - - public T this[string key] - { - get { return OpenEntry(entries[key]); } - } - - private T OpenEntry(ZipArchiveEntry entry) - { - T array; - if (arrays.TryGetValue(entry.FullName, out array)) - return array; - - using (Stream s = entry.Open()) - { - array = Load_Npz(s); - arrays[entry.FullName] = array; - return array; - } - } - - protected virtual T Load_Npz(Stream s) - { - return np.Load(s); - } - - public bool ContainsKey(string key) - { - return entries.ContainsKey(key); - } - - public bool TryGetValue(string key, out T value) - { - value = default(T); - ZipArchiveEntry entry; - if (!entries.TryGetValue(key, out entry)) - return false; - value = OpenEntry(entry); - return true; - } - - public IEnumerator> GetEnumerator() - { - foreach (var entry in archive.Entries) - yield return new KeyValuePair(entry.FullName, OpenEntry(entry)); - } - - IEnumerator IEnumerable.GetEnumerator() - { - foreach (var entry in archive.Entries) - yield return new KeyValuePair(entry.FullName, OpenEntry(entry)); - } - - IEnumerator IEnumerable.GetEnumerator() - { - foreach (var entry in archive.Entries) - yield return OpenEntry(entry); - } - - public void CopyTo(Array array, int arrayIndex) - { - foreach (var v in this) - array.SetValue(v, arrayIndex++); - } - - public void CopyTo(T[] array, int arrayIndex) - { - foreach (var v in this) - array.SetValue(v, arrayIndex++); - } - - public void Add(T item) - { - throw new ReadOnlyException(); - } - - public void Clear() - { - throw new ReadOnlyException(); - } - - public bool Contains(T item) - { - foreach (var v in this) - if (Object.Equals(v.Value, item)) - return true; - return false; - } - - public bool Remove(T item) - { - throw new ReadOnlyException(); - } - - protected virtual void Dispose(bool disposing) - { - if (!disposedValue) - { - if (disposing) - { - archive.Dispose(); - stream.Dispose(); - } - - archive = null; - stream = null; - entries = null; - arrays = null; - - disposedValue = true; - } - } - - public void Dispose() - { - Dispose(true); - } - } - - public class NpzDictionary : NpzDictionary - { - bool jagged; - - public NpzDictionary(Stream stream, bool jagged) - : base(stream) - { - this.jagged = jagged; - } - - protected override Array Load_Npz(Stream s) - { - //if (jagged) - //return np.LoadJagged(s); - return np.LoadMatrix(s); - } - } -} diff --git a/src/TensorFlowNET.Core/Numpy/Numpy.Creation.cs b/src/TensorFlowNET.Core/Numpy/Numpy.Creation.cs index 9604392c1..409e5e310 100644 --- a/src/TensorFlowNET.Core/Numpy/Numpy.Creation.cs +++ b/src/TensorFlowNET.Core/Numpy/Numpy.Creation.cs @@ -1,8 +1,4 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.IO; -using System.Text; +using System.IO; using static Tensorflow.Binding; namespace Tensorflow.NumPy @@ -65,6 +61,7 @@ public static NDArray linspace(T start, T stop, int num = 50, bool endpoint = [AutoNumPy] public static NDArray load(string file) => tf.numpy.load(file); + [AutoNumPy] public static T Load(string path) where T : class, ICloneable, IList, ICollection, IEnumerable, IStructuralComparable, IStructuralEquatable { @@ -102,6 +99,7 @@ public static NDArray ndarray(Shape shape, TF_DataType dtype = TF_DataType.TF_DO public static NDArray ones(Shape shape, TF_DataType dtype = TF_DataType.TF_DOUBLE) => new NDArray(tf.ones(shape, dtype: dtype)); + [AutoNumPy] public static NDArray ones_like(NDArray a, TF_DataType dtype = TF_DataType.DtInvalid) => new NDArray(tf.ones_like(a, dtype: dtype)); diff --git a/src/TensorFlowNET.Core/Numpy/Numpy.cs b/src/TensorFlowNET.Core/Numpy/Numpy.cs index cd9373d46..72d2e981c 100644 --- a/src/TensorFlowNET.Core/Numpy/Numpy.cs +++ b/src/TensorFlowNET.Core/Numpy/Numpy.cs @@ -14,65 +14,58 @@ You may obtain a copy of the License at limitations under the License. ******************************************************************************/ -using System; -using System.Collections; -using System.Collections.Generic; -using System.Numerics; -using System.Text; +namespace Tensorflow.NumPy; -namespace Tensorflow.NumPy +public partial class np { - public partial class np - { - /// - /// A convenient alias for None, useful for indexing arrays. - /// - /// https://docs.scipy.org/doc/numpy-1.17.0/reference/arrays.indexing.html



https://stackoverflow.com/questions/42190783/what-does-three-dots-in-python-mean-when-indexing-what-looks-like-a-number - public static readonly Slice newaxis = new Slice(null, null, 1) { IsNewAxis = true }; + /// + /// A convenient alias for None, useful for indexing arrays. + /// + /// https://docs.scipy.org/doc/numpy-1.17.0/reference/arrays.indexing.html



https://stackoverflow.com/questions/42190783/what-does-three-dots-in-python-mean-when-indexing-what-looks-like-a-number + public static readonly Slice newaxis = new Slice(null, null, 1) { IsNewAxis = true }; - // https://docs.scipy.org/doc/numpy-1.16.0/user/basics.types.html - #region data type - public static readonly TF_DataType @bool = TF_DataType.TF_BOOL; - public static readonly TF_DataType @char = TF_DataType.TF_INT8; - public static readonly TF_DataType @byte = TF_DataType.TF_INT8; - public static readonly TF_DataType uint8 = TF_DataType.TF_UINT8; - public static readonly TF_DataType ubyte = TF_DataType.TF_UINT8; - public static readonly TF_DataType int16 = TF_DataType.TF_INT16; - public static readonly TF_DataType uint16 = TF_DataType.TF_UINT16; - public static readonly TF_DataType int32 = TF_DataType.TF_INT32; - public static readonly TF_DataType uint32 = TF_DataType.TF_UINT32; - public static readonly TF_DataType int64 = TF_DataType.TF_INT64; - public static readonly TF_DataType uint64 = TF_DataType.TF_UINT64; - public static readonly TF_DataType float32 = TF_DataType.TF_FLOAT; - public static readonly TF_DataType float64 = TF_DataType.TF_DOUBLE; - public static readonly TF_DataType @double = TF_DataType.TF_DOUBLE; - public static readonly TF_DataType @decimal = TF_DataType.TF_DOUBLE; - public static readonly TF_DataType complex_ = TF_DataType.TF_COMPLEX; - public static readonly TF_DataType complex64 = TF_DataType.TF_COMPLEX64; - public static readonly TF_DataType complex128 = TF_DataType.TF_COMPLEX128; - #endregion + // https://docs.scipy.org/doc/numpy-1.16.0/user/basics.types.html + #region data type + public static readonly TF_DataType @bool = TF_DataType.TF_BOOL; + public static readonly TF_DataType @char = TF_DataType.TF_INT8; + public static readonly TF_DataType @byte = TF_DataType.TF_INT8; + public static readonly TF_DataType uint8 = TF_DataType.TF_UINT8; + public static readonly TF_DataType ubyte = TF_DataType.TF_UINT8; + public static readonly TF_DataType int16 = TF_DataType.TF_INT16; + public static readonly TF_DataType uint16 = TF_DataType.TF_UINT16; + public static readonly TF_DataType int32 = TF_DataType.TF_INT32; + public static readonly TF_DataType uint32 = TF_DataType.TF_UINT32; + public static readonly TF_DataType int64 = TF_DataType.TF_INT64; + public static readonly TF_DataType uint64 = TF_DataType.TF_UINT64; + public static readonly TF_DataType float32 = TF_DataType.TF_FLOAT; + public static readonly TF_DataType float64 = TF_DataType.TF_DOUBLE; + public static readonly TF_DataType @double = TF_DataType.TF_DOUBLE; + public static readonly TF_DataType @decimal = TF_DataType.TF_DOUBLE; + public static readonly TF_DataType complex_ = TF_DataType.TF_COMPLEX; + public static readonly TF_DataType complex64 = TF_DataType.TF_COMPLEX64; + public static readonly TF_DataType complex128 = TF_DataType.TF_COMPLEX128; + #endregion - public static double nan => double.NaN; - public static double NAN => double.NaN; - public static double NaN => double.NaN; - public static double pi => Math.PI; - public static double e => Math.E; - public static double euler_gamma => 0.57721566490153286060651209008240243d; - public static double inf => double.PositiveInfinity; - public static double infty => double.PositiveInfinity; - public static double Inf => double.PositiveInfinity; - public static double NINF => double.NegativeInfinity; - public static double PINF => double.PositiveInfinity; - public static double Infinity => double.PositiveInfinity; - public static double infinity => double.PositiveInfinity; + public static double nan => double.NaN; + public static double NAN => double.NaN; + public static double NaN => double.NaN; + public static double pi => Math.PI; + public static double e => Math.E; + public static double euler_gamma => 0.57721566490153286060651209008240243d; + public static double inf => double.PositiveInfinity; + public static double infty => double.PositiveInfinity; + public static double Inf => double.PositiveInfinity; + public static double NINF => double.NegativeInfinity; + public static double PINF => double.PositiveInfinity; + public static double Infinity => double.PositiveInfinity; + public static double infinity => double.PositiveInfinity; - public static bool array_equal(NDArray a, NDArray b) - => a.Equals(b); + public static bool array_equal(NDArray a, NDArray b) + => a.Equals(b); - public static bool allclose(NDArray a, NDArray b, double rtol = 1.0E-5, double atol = 1.0E-8, - bool equal_nan = false) => throw new NotImplementedException(""); + public static bool allclose(NDArray a, NDArray b, double rtol = 1.0E-5, double atol = 1.0E-8, + bool equal_nan = false) => throw new NotImplementedException(""); - public static RandomizedImpl random = new RandomizedImpl(); - public static LinearAlgebraImpl linalg = new LinearAlgebraImpl(); - } + public static RandomizedImpl random = new RandomizedImpl(); + public static LinearAlgebraImpl linalg = new LinearAlgebraImpl(); } diff --git a/test/TensorFlowNET.UnitTest/NumPy/Persistence.Test.cs b/test/TensorFlowNET.UnitTest/NumPy/Persistence.Test.cs new file mode 100644 index 000000000..21db6acc0 --- /dev/null +++ b/test/TensorFlowNET.UnitTest/NumPy/Persistence.Test.cs @@ -0,0 +1,42 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Tensorflow.NumPy; + +namespace TensorFlowNET.UnitTest.NumPy; + +/// +/// https://numpy.org/doc/stable/reference/generated/numpy.save.html +/// +[TestClass] +public class PersistenceTest : EagerModeTestBase +{ + [TestMethod] + public void SaveNpy() + { + var x = np.arange(10f).reshape((2, 5)); + np.save("arange.npy", x); + + var x2 = np.load("arange.npy"); + Assert.AreEqual(x.shape, x2.shape); + } + + [TestMethod] + public void SaveNpz() + { + var x = np.arange(10f).reshape((2, 5)); + var y = np.arange(10f).reshape((5, 2)); + + np.savez("arange.npz", x, y); + var z = np.loadz("arange.npz"); + + np.savez("arange_named.npz", new { x, y }); + z = np.loadz("arange_named.npz"); + Assert.AreEqual(z["x"].shape, x.shape); + Assert.AreEqual(z["y"].shape, y.shape); + + np.savez_compressed("arange_compressed.npz", x, y); + np.savez_compressed("arange_compressed_named.npz", new { x, y }); + z = np.loadz("arange_compressed_named.npz"); + Assert.AreEqual(z["x"].shape, x.shape); + Assert.AreEqual(z["y"].shape, y.shape); + } +} From e5dc65a7d927e2a3dec34cad3568a555dbe282b5 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sun, 26 Feb 2023 11:49:25 -0600 Subject: [PATCH 43/52] Support SigmoidFocalCrossEntropy, better for imbalanced multi-class task. --- .../Keras/Losses/ILossesApi.cs | 15 ++++++ src/TensorFlowNET.Keras/Losses/LossesApi.cs | 11 +++++ .../Losses/SigmoidFocalCrossEntropy.cs | 48 +++++++++++++++++++ .../Losses/LossesTest.cs | 14 ++++++ 4 files changed, 88 insertions(+) create mode 100644 src/TensorFlowNET.Keras/Losses/SigmoidFocalCrossEntropy.cs diff --git a/src/TensorFlowNET.Core/Keras/Losses/ILossesApi.cs b/src/TensorFlowNET.Core/Keras/Losses/ILossesApi.cs index c42493368..4c92512d4 100644 --- a/src/TensorFlowNET.Core/Keras/Losses/ILossesApi.cs +++ b/src/TensorFlowNET.Core/Keras/Losses/ILossesApi.cs @@ -38,4 +38,19 @@ ILossFunc Huber(string reduction = null, ILossFunc LogCosh(string reduction = null, string name = null); + + /// + /// Implements the focal loss function. + /// + /// + /// + /// + /// + /// + /// + ILossFunc SigmoidFocalCrossEntropy(bool from_logits = false, + float alpha = 0.25f, + float gamma = 2.0f, + string reduction = "none", + string name = "sigmoid_focal_crossentropy"); } diff --git a/src/TensorFlowNET.Keras/Losses/LossesApi.cs b/src/TensorFlowNET.Keras/Losses/LossesApi.cs index 29e15e53c..79f16a2eb 100644 --- a/src/TensorFlowNET.Keras/Losses/LossesApi.cs +++ b/src/TensorFlowNET.Keras/Losses/LossesApi.cs @@ -37,5 +37,16 @@ public ILossFunc Huber(string reduction = null, string name = null, Tensor delta public ILossFunc LogCosh(string reduction = null, string name = null) => new LogCosh(reduction: reduction, name: name); + + public ILossFunc SigmoidFocalCrossEntropy(bool from_logits = false, + float alpha = 0.25F, + float gamma = 2, + string reduction = "none", + string name = "sigmoid_focal_crossentropy") + => new SigmoidFocalCrossEntropy(from_logits: from_logits, + alpha: alpha, + gamma: gamma, + reduction: reduction, + name: name); } } diff --git a/src/TensorFlowNET.Keras/Losses/SigmoidFocalCrossEntropy.cs b/src/TensorFlowNET.Keras/Losses/SigmoidFocalCrossEntropy.cs new file mode 100644 index 000000000..7ac3fa0bb --- /dev/null +++ b/src/TensorFlowNET.Keras/Losses/SigmoidFocalCrossEntropy.cs @@ -0,0 +1,48 @@ +using static HDF.PInvoke.H5L.info_t; + +namespace Tensorflow.Keras.Losses; + +public class SigmoidFocalCrossEntropy : LossFunctionWrapper, ILossFunc +{ + float _alpha; + float _gamma; + + public SigmoidFocalCrossEntropy(bool from_logits = false, + float alpha = 0.25f, + float gamma = 2.0f, + string reduction = "none", + string name = "sigmoid_focal_crossentropy") : + base(reduction: reduction, + name: name, + from_logits: from_logits) + { + _alpha = alpha; + _gamma = gamma; + } + + + public override Tensor Apply(Tensor y_true, Tensor y_pred, bool from_logits = false, int axis = -1) + { + y_true = tf.cast(y_true, dtype: y_pred.dtype); + var ce = keras.backend.binary_crossentropy(y_true, y_pred, from_logits: from_logits); + var pred_prob = from_logits ? tf.sigmoid(y_pred) : y_pred; + + var p_t = (y_true * pred_prob) + ((1f - y_true) * (1f - pred_prob)); + Tensor alpha_factor = constant_op.constant(1.0f); + Tensor modulating_factor = constant_op.constant(1.0f); + + if(_alpha > 0) + { + var alpha = tf.cast(constant_op.constant(_alpha), dtype: y_true.dtype); + alpha_factor = y_true * alpha + (1f - y_true) * (1f - alpha); + } + + if (_gamma > 0) + { + var gamma = tf.cast(constant_op.constant(_gamma), dtype: y_true.dtype); + modulating_factor = tf.pow(1f - p_t, gamma); + } + + return tf.reduce_sum(alpha_factor * modulating_factor * ce, axis = -1); + } +} diff --git a/test/TensorFlowNET.Keras.UnitTest/Losses/LossesTest.cs b/test/TensorFlowNET.Keras.UnitTest/Losses/LossesTest.cs index b19f0203a..98fa1de12 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Losses/LossesTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Losses/LossesTest.cs @@ -5,6 +5,7 @@ using System.Text; using System.Threading.Tasks; using Tensorflow; +using Tensorflow.NumPy; using TensorFlowNET.Keras.UnitTest; using static Tensorflow.Binding; using static Tensorflow.KerasApi; @@ -48,4 +49,17 @@ public void BinaryCrossentropy() loss = bce.Call(y_true, y_pred); Assert.AreEqual(new float[] { 0.23515666f, 1.4957594f}, loss.numpy()); } + + /// + /// https://www.tensorflow.org/addons/api_docs/python/tfa/losses/SigmoidFocalCrossEntropy + /// + [TestMethod] + public void SigmoidFocalCrossEntropy() + { + var y_true = np.expand_dims(np.array(new[] { 1.0f, 1.0f, 0 })); + var y_pred = np.expand_dims(np.array(new[] { 0.97f, 0.91f, 0.03f })); + var bce = tf.keras.losses.SigmoidFocalCrossEntropy(); + var loss = bce.Call(y_true, y_pred); + Assert.AreEqual(new[] { 6.8532745e-06f, 1.909787e-04f, 2.0559824e-05f }, loss.numpy()); + } } From 3b0cdd878ce8d1e1120acd99b8e77401e06a65bb Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sun, 26 Feb 2023 12:29:57 -0600 Subject: [PATCH 44/52] Fix double data type issue in hamming_loss_fn. --- src/TensorFlowNET.Keras/Metrics/metrics_utils.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs b/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs index 69cc789e9..269bb1fb2 100644 --- a/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs +++ b/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs @@ -31,7 +31,7 @@ public static Tensor hamming_loss_fn(Tensor y_true, Tensor y_pred, Tensor thresh threshold = tf.reduce_max(y_pred, axis: -1, keepdims: true); // make sure [0, 0, 0] doesn't become [1, 1, 1] // Use abs(x) > eps, instead of x != 0 to check for zero - y_pred = tf.logical_and(y_pred >= threshold, tf.abs(y_pred) > 1e-12); + y_pred = tf.logical_and(y_pred >= threshold, tf.abs(y_pred) > 1e-12f); } else { From 21090633863bf6aacfc20d6ca93531833d0be5e8 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Sun, 26 Feb 2023 12:59:00 -0600 Subject: [PATCH 45/52] Fix FBetaScore data type issue. --- src/TensorFlowNET.Keras/Metrics/FBetaScore.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TensorFlowNET.Keras/Metrics/FBetaScore.cs b/src/TensorFlowNET.Keras/Metrics/FBetaScore.cs index 39e3e9af9..a40a7caa9 100644 --- a/src/TensorFlowNET.Keras/Metrics/FBetaScore.cs +++ b/src/TensorFlowNET.Keras/Metrics/FBetaScore.cs @@ -53,7 +53,7 @@ public override Tensor update_state(Tensor y_true, Tensor y_pred, Tensor sample_ _threshold = tf.reduce_max(y_pred, axis: -1, keepdims: true); // make sure [0, 0, 0] doesn't become [1, 1, 1] // Use abs(x) > eps, instead of x != 0 to check for zero - y_pred = tf.logical_and(y_pred >= _threshold, tf.abs(y_pred) > 1e-12); + y_pred = tf.logical_and(y_pred >= _threshold, tf.abs(y_pred) > 1e-12f); } else { From 45f26269d807cf605e056cd24bcb4226a412ff11 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Mon, 27 Feb 2023 08:18:01 -0600 Subject: [PATCH 46/52] Add SparseCategoricalCrossentropy and SparseCategoricalAccuracy. --- .../Keras/Metrics/IMetricsApi.cs | 40 +++++++++++++ src/TensorFlowNET.Keras/BackendImpl.cs | 58 +++++++++++++++++++ src/TensorFlowNET.Keras/Metrics/MetricsApi.cs | 14 +++++ .../Metrics/SparseCategoricalAccuracy.cs | 11 ++++ .../Metrics/SparseCategoricalCrossentropy.cs | 16 +++++ .../Metrics/SparseTopKCategoricalAccuracy.cs | 11 ++++ .../Metrics/metrics_utils.cs | 2 +- .../Metrics/MetricsTest.cs | 54 +++++++++++++++++ 8 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 src/TensorFlowNET.Keras/Metrics/SparseCategoricalAccuracy.cs create mode 100644 src/TensorFlowNET.Keras/Metrics/SparseCategoricalCrossentropy.cs create mode 100644 src/TensorFlowNET.Keras/Metrics/SparseTopKCategoricalAccuracy.cs diff --git a/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs b/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs index 5d08cc78e..dbe4ac3fd 100644 --- a/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs +++ b/src/TensorFlowNET.Core/Keras/Metrics/IMetricsApi.cs @@ -22,6 +22,20 @@ Tensor categorical_crossentropy(Tensor y_true, Tensor y_pred, /// Sparse categorical accuracy values. Tensor sparse_categorical_accuracy(Tensor y_true, Tensor y_pred); + /// + /// Computes the sparse categorical crossentropy loss. + /// + /// + /// + /// + /// + /// + /// + Tensor sparse_categorical_crossentropy(Tensor y_true, Tensor y_pred, + bool from_logits = false, + int? ignore_class = null, + Axis? axis = null); + /// /// Computes how often targets are in the top `K` predictions. /// @@ -56,6 +70,16 @@ IMetricFunc CategoricalCrossentropy(string name = "categorical_crossentropy", float label_smoothing = 0f, Axis? axis = null); + /// + /// Computes the crossentropy metric between the labels and predictions. + /// + /// + IMetricFunc SparseCategoricalCrossentropy(string name = "sparse_categorical_crossentropy", + TF_DataType dtype = TF_DataType.TF_FLOAT, + bool from_logits = false, + int? ignore_class = null, + Axis? axis = null); + /// /// Computes the crossentropy metric between the labels and predictions. /// @@ -63,6 +87,13 @@ IMetricFunc CategoricalCrossentropy(string name = "categorical_crossentropy", IMetricFunc CategoricalAccuracy(string name = "categorical_accuracy", TF_DataType dtype = TF_DataType.TF_FLOAT); + /// + /// Calculates how often predictions match integer labels. + /// + /// + IMetricFunc SparseCategoricalAccuracy(string name = "sparse_categorical_accuracy", + TF_DataType dtype = TF_DataType.TF_FLOAT); + /// /// Computes the cosine similarity between the labels and predictions. /// @@ -114,6 +145,15 @@ IMetricFunc TopKCategoricalAccuracy(int k = 5, string name = "top_k_categorical_accuracy", TF_DataType dtype = TF_DataType.TF_FLOAT); + /// + /// Computes how often integer targets are in the top K predictions. + /// + /// + /// + IMetricFunc SparseTopKCategoricalAccuracy(int k = 5, + string name = "sparse_top_k_categorical_accuracy", + TF_DataType dtype = TF_DataType.TF_FLOAT); + /// /// Computes the precision of the predictions with respect to the labels. /// diff --git a/src/TensorFlowNET.Keras/BackendImpl.cs b/src/TensorFlowNET.Keras/BackendImpl.cs index 0c9da015b..c49fc1409 100644 --- a/src/TensorFlowNET.Keras/BackendImpl.cs +++ b/src/TensorFlowNET.Keras/BackendImpl.cs @@ -276,6 +276,64 @@ public Tensor categorical_crossentropy(Tensor target, Tensor output, bool from_l return -math_ops.reduce_sum(target * math_ops.log(output), new Axis(axis)); } + public Tensor sparse_categorical_crossentropy(Tensor target, Tensor output, bool from_logits = false, int axis = -1, int? ignore_class = null) + { + target = tf.cast(target, tf.int64); + if (!from_logits) + { + var epsilon_ = constant_op.constant(epsilon(), output.dtype.as_base_dtype()); + output = tf.clip_by_value(output, epsilon_, 1 - epsilon_); + output = tf.math.log(output); + } + var output_rank = output.shape.ndim; + if (output_rank > -1) + { + axis = Math.Abs(axis) % output_rank; + if (axis != output_rank - 1) + { + /*var permutation = list( + itertools.chain( + range(axis), range(axis + 1, output_rank), [axis] + ) + ); + output = tf.transpose(output, perm: permutation);*/ + throw new NotImplementedException(""); + } + + } + + var output_shape = tf.shape(output); + var target_rank = target.shape.ndim; + var update_shape = target_rank > -1 && output_rank > -1 && target_rank != output_rank - 1; + if (update_shape) + { + /*var target = flatten(target); + output = tf.reshape(output, [-1, output_shape[-1]]);*/ + throw new NotImplementedException(""); + } + + if (ignore_class.HasValue) + { + throw new NotImplementedException(""); + } + + var res = tf.nn.sparse_softmax_cross_entropy_with_logits(labels: target, logits: output); + + if (ignore_class.HasValue) + { + throw new NotImplementedException(""); + } + + if (update_shape && output_rank >= 3) + { + // If our output includes timesteps or + // spatial dimensions we need to reshape + res = tf.reshape(res, output_shape[":-1"]); + } + + return res; + } + public Tensor binary_crossentropy(Tensor target, Tensor output, bool from_logits = false) { if (from_logits) diff --git a/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs b/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs index 585fefae2..e3881cf1a 100644 --- a/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs +++ b/src/TensorFlowNET.Keras/Metrics/MetricsApi.cs @@ -27,6 +27,11 @@ public Tensor categorical_crossentropy(Tensor y_true, Tensor y_pred, bool from_l return keras.backend.categorical_crossentropy(y_true, y_pred, from_logits: from_logits, axis: axis); } + public Tensor sparse_categorical_crossentropy(Tensor y_true, Tensor y_pred, bool from_logits = false, int? ignore_class = null, Axis? axis = null) + { + return keras.backend.sparse_categorical_crossentropy(y_true, y_pred, from_logits: from_logits, axis: axis ?? -1, ignore_class: ignore_class); + } + /// /// Calculates how often predictions matches integer labels. /// @@ -103,5 +108,14 @@ public IMetricFunc Precision(float thresholds = 0.5f, int top_k = 0, int class_i public IMetricFunc Recall(float thresholds = 0.5f, int top_k = 0, int class_id = 0, string name = "recall", TF_DataType dtype = TF_DataType.TF_FLOAT) => new Recall(thresholds: thresholds, top_k: top_k, class_id: class_id, name: name, dtype: dtype); + + public IMetricFunc SparseCategoricalCrossentropy(string name = "sparse_categorical_crossentropy", TF_DataType dtype = TF_DataType.TF_FLOAT, bool from_logits = false, int? ignore_class = null, Axis? axis = null) + => new SparseCategoricalCrossentropy(name: name, dtype: dtype, from_logits: from_logits, ignore_class: ignore_class, axis: axis ?? -1); + + public IMetricFunc SparseTopKCategoricalAccuracy(int k = 5, string name = "sparse_top_k_categorical_accuracy", TF_DataType dtype = TF_DataType.TF_FLOAT) + => new SparseTopKCategoricalAccuracy(k: k, name: name, dtype: dtype); + + public IMetricFunc SparseCategoricalAccuracy(string name = "sparse_categorical_accuracy", TF_DataType dtype = TF_DataType.TF_FLOAT) + => new SparseCategoricalAccuracy(name: name, dtype: dtype); } } diff --git a/src/TensorFlowNET.Keras/Metrics/SparseCategoricalAccuracy.cs b/src/TensorFlowNET.Keras/Metrics/SparseCategoricalAccuracy.cs new file mode 100644 index 000000000..6cad9aac3 --- /dev/null +++ b/src/TensorFlowNET.Keras/Metrics/SparseCategoricalAccuracy.cs @@ -0,0 +1,11 @@ +namespace Tensorflow.Keras.Metrics; + +public class SparseCategoricalAccuracy : MeanMetricWrapper +{ + public SparseCategoricalAccuracy(string name = "sparse_categorical_accuracy", TF_DataType dtype = TF_DataType.TF_FLOAT) + : base((yt, yp) => metrics_utils.sparse_categorical_matches(yt, yp), + name: name, + dtype: dtype) + { + } +} diff --git a/src/TensorFlowNET.Keras/Metrics/SparseCategoricalCrossentropy.cs b/src/TensorFlowNET.Keras/Metrics/SparseCategoricalCrossentropy.cs new file mode 100644 index 000000000..d517da913 --- /dev/null +++ b/src/TensorFlowNET.Keras/Metrics/SparseCategoricalCrossentropy.cs @@ -0,0 +1,16 @@ +namespace Tensorflow.Keras.Metrics; + +public class SparseCategoricalCrossentropy : MeanMetricWrapper +{ + public SparseCategoricalCrossentropy(string name = "sparse_categorical_crossentropy", + TF_DataType dtype = TF_DataType.TF_FLOAT, + bool from_logits = false, + int? ignore_class = null, + Axis? axis = null) + : base((yt, yp) => keras.metrics.sparse_categorical_crossentropy( + yt, yp, from_logits: from_logits, ignore_class: ignore_class, axis: axis ?? -1), + name: name, + dtype: dtype) + { + } +} diff --git a/src/TensorFlowNET.Keras/Metrics/SparseTopKCategoricalAccuracy.cs b/src/TensorFlowNET.Keras/Metrics/SparseTopKCategoricalAccuracy.cs new file mode 100644 index 000000000..eb6d9f3b3 --- /dev/null +++ b/src/TensorFlowNET.Keras/Metrics/SparseTopKCategoricalAccuracy.cs @@ -0,0 +1,11 @@ +namespace Tensorflow.Keras.Metrics; + +public class SparseTopKCategoricalAccuracy : MeanMetricWrapper +{ + public SparseTopKCategoricalAccuracy(int k = 5, string name = "sparse_top_k_categorical_accuracy", TF_DataType dtype = TF_DataType.TF_FLOAT) + : base((yt, yp) => metrics_utils.sparse_top_k_categorical_matches(yt, yp, k), + name: name, + dtype: dtype) + { + } +} diff --git a/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs b/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs index 269bb1fb2..be6a49ec5 100644 --- a/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs +++ b/src/TensorFlowNET.Keras/Metrics/metrics_utils.cs @@ -73,7 +73,7 @@ public static Tensor sparse_categorical_matches(Tensor y_true, Tensor y_pred) y_true = tf.squeeze(y_true, new Shape(-1)); } y_pred = tf.math.argmax(y_pred, axis: -1); - + y_pred = tf.cast(y_pred, y_true.dtype); var matches = tf.cast( tf.equal(y_true, y_pred), dtype: keras.backend.floatx() diff --git a/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs b/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs index 267cef815..04810db31 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Metrics/MetricsTest.cs @@ -74,6 +74,26 @@ public void CategoricalAccuracy() Assert.AreEqual(r, 0.3f); } + /// + /// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/SparseCategoricalAccuracy + /// + [TestMethod] + public void SparseCategoricalAccuracy() + { + var y_true = np.array(new[] { 2, 1 }); + var y_pred = np.array(new[,] { { 0.1f, 0.6f, 0.3f }, { 0.05f, 0.95f, 0f } }); + var m = tf.keras.metrics.SparseCategoricalAccuracy(); + m.update_state(y_true, y_pred); + var r = m.result().numpy(); + Assert.AreEqual(r, 0.5f); + + m.reset_states(); + var weights = np.array(new[] { 0.7f, 0.3f }); + m.update_state(y_true, y_pred, sample_weight: weights); + r = m.result().numpy(); + Assert.AreEqual(r, 0.3f); + } + /// /// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/CategoricalCrossentropy /// @@ -94,6 +114,20 @@ public void CategoricalCrossentropy() Assert.AreEqual(r, 1.6271976f); } + /// + /// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/SparseCategoricalCrossentropy + /// + [TestMethod] + public void SparseCategoricalCrossentropy() + { + var y_true = np.array(new[] { 1, 2 }); + var y_pred = np.array(new[,] { { 0.05f, 0.95f, 0f }, { 0.1f, 0.8f, 0.1f } }); + var m = tf.keras.metrics.SparseCategoricalCrossentropy(); + m.update_state(y_true, y_pred); + var r = m.result().numpy(); + Assert.AreEqual(r, 1.1769392f); + } + /// /// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/CosineSimilarity /// @@ -207,6 +241,26 @@ public void TopKCategoricalAccuracy() Assert.AreEqual(r, 0.3f); } + /// + /// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/SparseTopKCategoricalAccuracy + /// + [TestMethod] + public void SparseTopKCategoricalAccuracy() + { + var y_true = np.array(new[] { 2, 1 }); + var y_pred = np.array(new[,] { { 0.1f, 0.9f, 0.8f }, { 0.05f, 0.95f, 0f } }); + var m = tf.keras.metrics.SparseTopKCategoricalAccuracy(k: 1); + m.update_state(y_true, y_pred); + var r = m.result().numpy(); + Assert.AreEqual(r, 0.5f); + + m.reset_states(); + var weights = np.array(new[] { 0.7f, 0.3f }); + m.update_state(y_true, y_pred, sample_weight: weights); + r = m.result().numpy(); + Assert.AreEqual(r, 0.3f); + } + /// /// https://www.tensorflow.org/api_docs/python/tf/keras/metrics/top_k_categorical_accuracy /// From 52b513d750ca60b4efb64e183ec38bf4f4c58527 Mon Sep 17 00:00:00 2001 From: Yaohui Liu Date: Fri, 3 Mar 2023 11:52:52 +0800 Subject: [PATCH 47/52] Support loading of SavedModel format (#989) * Add CheckpointReader and corresponding C APIs. * Add essential components of SavedModel format loading. * Add checkpoint reading for SavedModel format loading. * Revise customized json converters. * Add support for loading models from python. * Fix the duplicated weights in Keras.Model. * Add alexnet loading test and check for loaded weights. * Fix ci error caused by branch merge. * Resolve the comments and errors. * Fix the stucking of training when loading model. * Fix the stucking of training when loading model. * fix intptr. --------- Co-authored-by: Haiping Chen --- .../Checkpoint/CheckPointUtils.cs | 18 + .../Checkpoint/CheckpointReader.cs | 100 +++ .../Checkpoint/SaveUtilV1.cs | 6 +- .../Checkpoint/c_api.checkpoint.cs | 27 + .../Checkpoint/checkpoint.cs | 378 +++++++++++ .../Checkpoint/functional_saver.cs | 2 +- src/TensorFlowNET.Core/Checkpoint/restore.cs | 331 +++++++++ src/TensorFlowNET.Core/Eager/execute.cs | 6 +- .../Functions/ConcreteFunction.cs | 15 +- src/TensorFlowNET.Core/IO/gfile.cs | 12 + .../Common/CustomizedAxisJsonConverter.cs | 11 +- .../Common/CustomizedDTypeJsonConverter.cs | 36 + .../CustomizedNodeConfigJsonConverter.cs | 37 +- .../Common/CustomizedShapeJsonConverter.cs | 24 +- src/TensorFlowNET.Core/Keras/Layers/ILayer.cs | 1 + .../Keras/Saving/ModelConfig.cs | 3 + .../ModelSaving/ModelSaver.cs | 1 + .../Operations/NnOps/RNNCell.cs | 1 + src/TensorFlowNET.Core/Operations/gen_ops.cs | 43 +- src/TensorFlowNET.Core/Operations/io_ops.cs | 1 + .../Operations/resource_variable_ops.cs | 2 +- src/TensorFlowNET.Core/Tensors/TF_DataType.cs | 6 +- src/TensorFlowNET.Core/Tensors/dtypes.cs | 3 + .../Training/Saving/SaveableObject.cs | 18 + .../Training/Saving/SavedModel/LoadOptions.cs | 23 + .../Saving/SavedModel/RevivedTypes.cs | 9 +- .../Saving/SavedModel}/SaveOptions.cs | 12 +- .../Saving/SavedModel/SaveableView.cs | 1 - .../Saving/SavedModel/WrapperFunction.cs | 22 + .../SavedModel/function_deserialization.cs | 36 + .../Training/Saving/SavedModel/loader.cs | 641 ++++++++++++++++++ .../Saving/SavedModel/loader.static.cs | 122 ++++ .../Training/Saving/SavedModel/save.cs | 1 - .../Saving/SavedModel/save_context.cs | 1 - .../Saving/saveable_object_util.py.cs | 94 ++- src/TensorFlowNET.Core/Training/Trackable.cs | 53 +- .../Training/TrackableUtils.cs | 7 +- .../Variables/BaseResourceVariable.cs | 14 +- .../Variables/ResourceVariable.cs | 18 + .../Engine/Functional.FromConfig.cs | 11 +- src/TensorFlowNET.Keras/Engine/Functional.cs | 14 +- .../Engine/Layer.Layers.cs | 26 + .../Engine/Layer.Serialize.cs | 2 +- src/TensorFlowNET.Keras/Engine/Layer.cs | 112 +-- src/TensorFlowNET.Keras/Engine/Model.Save.cs | 2 +- src/TensorFlowNET.Keras/Engine/Model.cs | 45 +- src/TensorFlowNET.Keras/Engine/Sequential.cs | 15 +- .../Layers/Activation/ELU.cs | 3 +- .../Layers/Activation/Exponential.cs | 3 +- .../Layers/Activation/SELU.cs | 3 +- src/TensorFlowNET.Keras/Layers/Core/Dense.cs | 5 - .../Layers/Core/InputLayer.cs | 5 - src/TensorFlowNET.Keras/Metrics/Metric.cs | 2 +- src/TensorFlowNET.Keras/Models/ModelsApi.cs | 16 +- .../Saving/KerasObjectLoader.cs | 561 ++++++++++++++- .../Saving/SavedModel/Save.cs | 6 +- .../Saving/SavedModel/load.cs | 96 +++ .../Saving/SavedModel/load_context.cs | 69 ++ .../Utils/generic_utils.cs | 68 ++ src/TensorFlowNET.Keras/Utils/layer_utils.cs | 2 +- .../simple_model_from_auto_compile/bias0.npy | Bin 0 -> 528 bytes .../fingerprint.pb | Bin 0 -> 55 bytes .../keras_metadata.pb | 9 + .../kernel1.npy | Bin 0 -> 4128 bytes .../saved_model.pb | Bin 0 -> 66811 bytes .../variables/variables.data-00000-of-00001 | Bin 0 -> 322030 bytes .../variables/variables.index | Bin 0 -> 620 bytes .../SaveModel/SequentialModelLoad.cs | 68 ++ ...ialModelTest.cs => SequentialModelSave.cs} | 24 +- .../Tensorflow.Keras.UnitTest.csproj | 24 + 70 files changed, 3118 insertions(+), 209 deletions(-) create mode 100644 src/TensorFlowNET.Core/Checkpoint/CheckpointReader.cs create mode 100644 src/TensorFlowNET.Core/Checkpoint/c_api.checkpoint.cs create mode 100644 src/TensorFlowNET.Core/Checkpoint/restore.cs create mode 100644 src/TensorFlowNET.Core/Keras/Common/CustomizedDTypeJsonConverter.cs create mode 100644 src/TensorFlowNET.Core/Training/Saving/SavedModel/LoadOptions.cs rename src/TensorFlowNET.Core/{ModelSaving => Training/Saving/SavedModel}/SaveOptions.cs (83%) create mode 100644 src/TensorFlowNET.Core/Training/Saving/SavedModel/WrapperFunction.cs create mode 100644 src/TensorFlowNET.Core/Training/Saving/SavedModel/function_deserialization.cs create mode 100644 src/TensorFlowNET.Core/Training/Saving/SavedModel/loader.cs create mode 100644 src/TensorFlowNET.Core/Training/Saving/SavedModel/loader.static.cs create mode 100644 src/TensorFlowNET.Keras/Saving/SavedModel/load.cs create mode 100644 src/TensorFlowNET.Keras/Saving/SavedModel/load_context.cs create mode 100644 test/TensorFlowNET.Keras.UnitTest/Assets/simple_model_from_auto_compile/bias0.npy create mode 100644 test/TensorFlowNET.Keras.UnitTest/Assets/simple_model_from_auto_compile/fingerprint.pb create mode 100644 test/TensorFlowNET.Keras.UnitTest/Assets/simple_model_from_auto_compile/keras_metadata.pb create mode 100644 test/TensorFlowNET.Keras.UnitTest/Assets/simple_model_from_auto_compile/kernel1.npy create mode 100644 test/TensorFlowNET.Keras.UnitTest/Assets/simple_model_from_auto_compile/saved_model.pb create mode 100644 test/TensorFlowNET.Keras.UnitTest/Assets/simple_model_from_auto_compile/variables/variables.data-00000-of-00001 create mode 100644 test/TensorFlowNET.Keras.UnitTest/Assets/simple_model_from_auto_compile/variables/variables.index create mode 100644 test/TensorFlowNET.Keras.UnitTest/SaveModel/SequentialModelLoad.cs rename test/TensorFlowNET.Keras.UnitTest/SaveModel/{SequentialModelTest.cs => SequentialModelSave.cs} (94%) diff --git a/src/TensorFlowNET.Core/Checkpoint/CheckPointUtils.cs b/src/TensorFlowNET.Core/Checkpoint/CheckPointUtils.cs index 8ae2dae8f..9793798d2 100644 --- a/src/TensorFlowNET.Core/Checkpoint/CheckPointUtils.cs +++ b/src/TensorFlowNET.Core/Checkpoint/CheckPointUtils.cs @@ -149,4 +149,22 @@ public static void add_checkpoint_values_check(TrackableObjectGraph object_graph // object_graph_proto.Nodes[i].has_checkpoint_values.value = checkpointed_trackables.Contains(i); // } } + + /// + /// Traverse the object graph and list all accessible objects. + /// + /// + public static IList list_objects(ObjectGraphView graph_view) + { + return objects_ids_and_slot_variables_and_paths(graph_view).Item1; + } + + internal static IEnumerable _objects_with_attributes(IEnumerable full_list) + { + return full_list.TakeWhile(x => + { + var saveables = x.gather_saveables_for_checkpoint(); + return saveables is not null && saveables.Count > 0; + }); + } } diff --git a/src/TensorFlowNET.Core/Checkpoint/CheckpointReader.cs b/src/TensorFlowNET.Core/Checkpoint/CheckpointReader.cs new file mode 100644 index 000000000..0cc8e5fbd --- /dev/null +++ b/src/TensorFlowNET.Core/Checkpoint/CheckpointReader.cs @@ -0,0 +1,100 @@ +using Tensorflow.Util; + +namespace Tensorflow.Checkpoint +{ + sealed class SafeCheckpointReaderHandle : SafeTensorflowHandle + { + public SafeCheckpointReaderHandle(): base() + { + + } + public SafeCheckpointReaderHandle(IntPtr handle): base(handle) + { + + } + + protected override bool ReleaseHandle() + { + c_api.TF_DeleteCheckpointReader(handle); + SetHandle(IntPtr.Zero); + return true; + } + } + public class CheckpointReader + { + private SafeCheckpointReaderHandle _handle; + public Dictionary VariableToDataTypeMap { get; set; } + public Dictionary VariableToShapeMap { get; set; } + + public CheckpointReader(string filename) + { + Status status = new Status(); + _handle = c_api.TF_NewCheckpointReader(filename, status.Handle); + status.Check(true); + ReadAllShapeAndType(); + } + + public int HasTensor(string name) + { + return c_api.TF_CheckpointReaderHasTensor(_handle, name); + } + + /// + /// Get the variable name. + /// + /// + /// + public string GetVariable(int index) + { + return c_api.StringPiece(c_api.TF_CheckpointReaderGetVariable(_handle, index)); + } + + public int Size() + { + return c_api.TF_CheckpointReaderSize(_handle); + } + + public TF_DataType GetVariableDataType(string name) + { + return c_api.TF_CheckpointReaderGetVariableDataType(_handle, name); + } + + public Shape GetVariableShape(string name) + { + int num_dims = GetVariableNumDims(name); + long[] dims = new long[num_dims]; + Status status = new Status(); + c_api.TF_CheckpointReaderGetVariableShape(_handle, name, dims, num_dims, status.Handle); + status.Check(true); + return new Shape(dims); + } + + public int GetVariableNumDims(string name) + { + return c_api.TF_CheckpointReaderGetVariableNumDims(_handle, name); + } + + public unsafe Tensor GetTensor(string name, TF_DataType dtype = TF_DataType.DtInvalid) + { + Status status = new Status(); + var tensor = c_api.TF_CheckpointReaderGetTensor(_handle, name, status.Handle); + status.Check(true); + return new Tensor(tensor); + } + + private void ReadAllShapeAndType() + { + VariableToDataTypeMap = new Dictionary(); + VariableToShapeMap = new Dictionary(); + int size = Size(); + for(int i = 0; i < size; i++) + { + var name = GetVariable(i); + var shape = GetVariableShape(name); + var dtype = GetVariableDataType(name); + VariableToDataTypeMap[name] = dtype; + VariableToShapeMap[name] = shape; + } + } + } +} diff --git a/src/TensorFlowNET.Core/Checkpoint/SaveUtilV1.cs b/src/TensorFlowNET.Core/Checkpoint/SaveUtilV1.cs index 3267ae126..72372e410 100644 --- a/src/TensorFlowNET.Core/Checkpoint/SaveUtilV1.cs +++ b/src/TensorFlowNET.Core/Checkpoint/SaveUtilV1.cs @@ -175,9 +175,9 @@ public static (IList, object?) generate_saveable_objects( { var name = factory_data.name; var key = factory_data.checkpoint_key; - var maybe_saveable = factory_data.factory; + var maybe_saveable = saveable_object_util.create_saveable_object(name, key, factory_data.factory); - // TODO: oneflow python has a process with callable `saveable_factory`. + // TODO: tensorflow python has a process with callable `saveable_factory`. List saveables = new(); if (maybe_saveable.TryGet(out var s)) { @@ -217,7 +217,7 @@ public static (IList, object?) generate_saveable_objects( public record class CheckpointFactoryData ( - Maybe factory, + Func> factory, string name, string checkpoint_key ); diff --git a/src/TensorFlowNET.Core/Checkpoint/c_api.checkpoint.cs b/src/TensorFlowNET.Core/Checkpoint/c_api.checkpoint.cs new file mode 100644 index 000000000..f956e3337 --- /dev/null +++ b/src/TensorFlowNET.Core/Checkpoint/c_api.checkpoint.cs @@ -0,0 +1,27 @@ +using System.Runtime.InteropServices; +using Tensorflow.Checkpoint; + +namespace Tensorflow +{ + public unsafe partial class c_api + { + [DllImport(TensorFlowLibName)] + internal static extern SafeCheckpointReaderHandle TF_NewCheckpointReader(string filename, SafeStatusHandle status); + [DllImport(TensorFlowLibName)] + internal static extern void TF_DeleteCheckpointReader(IntPtr reader); + [DllImport(TensorFlowLibName)] + internal static extern int TF_CheckpointReaderHasTensor(SafeCheckpointReaderHandle reader, string name); + [DllImport(TensorFlowLibName)] + internal static extern IntPtr TF_CheckpointReaderGetVariable(SafeCheckpointReaderHandle reader, int index); + [DllImport(TensorFlowLibName)] + internal static extern int TF_CheckpointReaderSize(SafeCheckpointReaderHandle reader); + [DllImport(TensorFlowLibName)] + internal static extern TF_DataType TF_CheckpointReaderGetVariableDataType(SafeCheckpointReaderHandle reader, string name); + [DllImport(TensorFlowLibName)] + internal static extern void TF_CheckpointReaderGetVariableShape(SafeCheckpointReaderHandle reader, string name, long[] dims, int num_dims, SafeStatusHandle status); + [DllImport(TensorFlowLibName)] + internal static extern int TF_CheckpointReaderGetVariableNumDims(SafeCheckpointReaderHandle reader, string name); + [DllImport(TensorFlowLibName)] + internal static extern SafeTensorHandle TF_CheckpointReaderGetTensor(SafeCheckpointReaderHandle reader, string name, SafeStatusHandle status); + } +} diff --git a/src/TensorFlowNET.Core/Checkpoint/checkpoint.cs b/src/TensorFlowNET.Core/Checkpoint/checkpoint.cs index 0c2862dac..1934ffd5f 100644 --- a/src/TensorFlowNET.Core/Checkpoint/checkpoint.cs +++ b/src/TensorFlowNET.Core/Checkpoint/checkpoint.cs @@ -6,8 +6,12 @@ using Tensorflow.Contexts; using Tensorflow.Eager; using Tensorflow.Train; +using Tensorflow.Exceptions; using static Tensorflow.TrackableObjectGraph.Types.TrackableObject.Types; using static Tensorflow.Binding; +using Tensorflow.Operations; +using Newtonsoft.Json; +using Tensorflow.Training; namespace Tensorflow.Checkpoint; @@ -21,8 +25,20 @@ public class TrackableSaver private TrackableObjectGraph _last_save_object_graph; private Tensor? _object_graph_feed_tensor = null; private Tensor? _file_prefix_feed_tensor = null; + private Tensor? _file_prefix_placeholder = null; private Dictionary? _object_map = null; private object? _cache = null; + public Tensor? FilePrefixPlaceHolder + { + get + { + return _file_prefix_placeholder; + } + set + { + _file_prefix_placeholder = value; + } + } public TrackableSaver(ObjectGraphView graph_view) { _graph_view = graph_view; @@ -192,4 +208,366 @@ public Tensor save(string file_prefix, int? checkpoint_number = null, Session? s return save_path; } } + + public LoadStatus restore(string? save_path, CheckpointOptions? options = null) + { + if (options is null) + { + options = new CheckpointOptions(); + } + if(save_path is null) + { + return new InitializationOnlyStatus(_graph_view, ops.uid()); + } + + CheckpointReader reader = new CheckpointReader(save_path); + bool graph_building = tf.Context.executing_eagerly(); + Dictionary dtype_map = null; + if (!graph_building) + { + dtype_map = reader.VariableToDataTypeMap; + } + Tensor object_graph_string = reader.GetTensor(Trackable.Constants.OBJECT_GRAPH_PROTO_KEY, dtype: TF_DataType.TF_STRING); + + Dictionary file_prefix_feed_dict; + Tensor file_prefix_tensor; + if (graph_building) + { + if(_file_prefix_placeholder is null) + { + tf.device("/cpu:0"); + _file_prefix_placeholder = constant_op.constant("model"); + } + file_prefix_tensor = _file_prefix_placeholder; + file_prefix_feed_dict = new(); + file_prefix_feed_dict[_file_prefix_placeholder] = save_path; + } + else + { + tf.device("/cpu:0"); + file_prefix_tensor = constant_op.constant(save_path); + file_prefix_feed_dict = null; + } + TrackableObjectGraph object_graph_proto = new(); + if(object_graph_string.ndim > 0) + { + object_graph_proto.MergeFrom(object_graph_string.BufferToArray()); + } + else + { + object_graph_proto.MergeFrom(object_graph_string.StringBytes()[0]); + } + CheckpointRestoreCoordinator checkpoint = new CheckpointRestoreCoordinator( + object_graph_proto: object_graph_proto, + save_path: save_path, + save_path_tensor: file_prefix_tensor, + reader: reader, + restore_op_cache: null, + graph_view: _graph_view, + options: options, + saveables_cache: null + ); + + new CheckpointPosition(checkpoint, 0).restore(_graph_view.Root); + + if(_graph_view.AttachedDependencies is not null) + { + foreach(var refer in _graph_view.AttachedDependencies) + { + if(refer.Name == "root") + { + continue; + } + int? proto_id = null; + // Find proto ID of attached dependency (if it is in the proto). + foreach (var proto_refer in object_graph_proto.Nodes[0].Children) + { + if(proto_refer.LocalName == refer.Name) + { + proto_id = proto_refer.NodeId; + break; + } + } + + if (proto_id is null) + { + continue; + } + + // Object has already been restored. This can happen when there's an + // indirect connection from the attached object to the root. + if (checkpoint.ObjectByProtoId.ContainsKey(proto_id.Value)) + { + continue; + } + + new CheckpointPosition(checkpoint, proto_id.Value).restore(refer.Refer); + } + } + + return new CheckpointLoadStatus(checkpoint, file_prefix_feed_dict, _graph_view); + } +} + +public class CheckpointRestoreCoordinator +{ + private CheckpointOptions _options; + private TrackableObjectGraph _object_graph_proto; + private int _restore_uid; + private HashSet _matched_proto_ids; + private Tensor _save_path_tensor; + private string _save_path_string; + private CheckpointReader _reader; + private Dictionary _dtype_map; + private Dictionary _shape_map; + private ObjectGraphView _graph_view; + private Dictionary> _slot_restorations; + private bool _expect_partial_attr; + private List _restore_ops; + private List _all_trackables; + private Dictionary _object_by_proto_id; + private Dictionary _restore_ops_by_name; + private Dictionary> _deferred_slot_restorations; + private Dictionary> _unused_attributes; + + public CheckpointRestoreCoordinator(TrackableObjectGraph object_graph_proto, string save_path, Tensor save_path_tensor, + CheckpointReader reader, object? restore_op_cache, ObjectGraphView graph_view, CheckpointOptions options, object? saveables_cache) + { + // TODO(Rinne): cache. + _options = options; + _object_graph_proto = object_graph_proto; + _restore_uid = ops.uid(); + _save_path_tensor = save_path_tensor; + _save_path_string = save_path; + _reader = reader; + if(_reader is null) + { + _reader = new CheckpointReader(save_path); + } + _dtype_map = _reader.VariableToDataTypeMap; + _shape_map = _reader.VariableToShapeMap; + _graph_view = graph_view; + _restore_ops = new List(); + _restore_ops_by_name = new Dictionary(); + _all_trackables = new List(); + _matched_proto_ids = new HashSet(); + _object_by_proto_id = new Dictionary(); + _slot_restorations = new Dictionary>(); + _deferred_slot_restorations = new Dictionary>(); + + _expect_partial_attr = false; + for(int i = 0; i < _object_graph_proto.Nodes.Count; i++) + { + var node = _object_graph_proto.Nodes[i]; + foreach(var slot_reference in node.SlotVariables) + { + _slot_restorations.SetDefault(slot_reference.OriginalVariableNodeId, new List()) + .Add(new SlotVariableRestoration(i, slot_reference.SlotVariableNodeId, slot_reference.SlotName)); + } + } + + // skip the deleter and cache. + } + + public bool ExpectPartial + { + get + { + return _expect_partial_attr; + } + set + { + _expect_partial_attr = value; + } + } + + /// + /// Corresponding to `all_python_objects` of tensorflow python + /// + public List AllTrackables => _all_trackables; + public HashSet MatchedProtoIds => _matched_proto_ids; + public Dictionary ObjectByProtoId => _object_by_proto_id; + public int RestoreUid => _restore_uid; + public TrackableObjectGraph ObjectGraphProto => _object_graph_proto; + public Dictionary> SlotRestorations => _slot_restorations; + public Dictionary> DeferredSlotRestorations => _deferred_slot_restorations; + public Dictionary RestoreOpsByName => _restore_ops_by_name; + public Dictionary> UnusedAttributes => _unused_attributes; + + public void new_restore_ops(IEnumerable new_ops) + { + _restore_ops.AddRange(new_ops); + // skip the callback. + } + + public List restore_saveables(Dictionary> tensor_saveables, List positions, object? registered_savers = null) + { + List restore_ops = new(); + foreach(var position in positions) + { + var key = position.ObjectProto.Attributes[0].CheckpointKey; + throw new NotImplementedException(); + } + + Dictionary variable_dict = new(); + foreach(var item in tensor_saveables) + { + if(item.Value.TryGet(out var variable)) + { + variable_dict[item.Key] = variable; + } + else + { + throw new TypeError(); + } + } + + if (tensor_saveables is not null && tensor_saveables.Count > 0) + { + var flat_saveables = saveable_object_util.validate_and_slice_inputs(variable_dict); + var new_restore_ops = MultiDeviceSaver.from_saveables(flat_saveables).restore(_save_path_tensor, _options); + if (!tf.Context.executing_eagerly()) + { + foreach(var item in new_restore_ops) + { + restore_ops.Add(item.Value); + Debug.Assert(!_restore_ops_by_name.ContainsKey(item.Key)); + _restore_ops_by_name[item.Key] = item.Value; + } + } + } + return restore_ops; + } +} + +public abstract class LoadStatus +{ + public abstract LoadStatus assert_consumed(); + public abstract LoadStatus assert_existing_objects_matched(); + public abstract LoadStatus assert_nontrivial_match(); + public abstract LoadStatus run_restore_ops(Session? session = null); + public abstract void initialize_or_restore(Session? session = null); + public virtual LoadStatus expect_partial() + { + return this; + } +} + +public class InitializationOnlyStatus: LoadStatus +{ + private int _restore_uid; + private ObjectGraphView _object_graph_view; + private Trackable _root; + public InitializationOnlyStatus(ObjectGraphView object_graph_view, int restore_uid) + { + _restore_uid = restore_uid; + _object_graph_view = object_graph_view; + _root = object_graph_view.Root; + } + public override LoadStatus assert_consumed() + { + throw new AssertionError("No checkpoint specified (save_path=None); nothing is being restored."); + } + public override LoadStatus assert_existing_objects_matched() + { + throw new AssertionError("No checkpoint specified (save_path=None); nothing is being restored."); + } + public override LoadStatus assert_nontrivial_match() + { + throw new AssertionError("No checkpoint specified (save_path=None); nothing is being restored."); + } + public override LoadStatus run_restore_ops(Session? session = null) + { + throw new AssertionError("No checkpoint specified, so no restore ops are available " + + "(save_path=None to Saver.restore)."); + } + public override void initialize_or_restore(Session? session = null) + { + if (tf.Context.executing_eagerly()) + { + return; + } + if(session is null) + { + session = new Session(); + } + var trackable_objects = CheckPointUtils.list_objects(_object_graph_view); + throw new NotImplementedException("Not implemented, please submit an issue to https://github.com/SciSharp/TensorFlow.NET/issues"); + } +} + +internal class CheckpointLoadStatus: LoadStatus +{ + private CheckpointRestoreCoordinator _checkpoint; + private Dictionary _feed_dict; + private ObjectGraphView _object_graph_view; + private Trackable _root; + public CheckpointLoadStatus(CheckpointRestoreCoordinator checkpoint, Dictionary feed_dict, ObjectGraphView graph_view):base() + { + _checkpoint = checkpoint; + _feed_dict = feed_dict; + _object_graph_view = graph_view; + _root = graph_view.Root; + } + + public CheckpointRestoreCoordinator Checkpoint => _checkpoint; + + public override LoadStatus assert_consumed() + { + throw new NotImplementedException(); + } + + public override LoadStatus assert_existing_objects_matched() + { + for(int i = 0; i < _checkpoint.ObjectGraphProto.Nodes.Count; i++) + { + var node = _checkpoint.ObjectGraphProto.Nodes[i]; + if(_checkpoint.ObjectByProtoId.TryGetValue(i, out var trackable) && + trackable.UpdateUid < _checkpoint.RestoreUid) + { + throw new AssertionError($"Object {node} not assigned a value from checkpoint."); + } + } + foreach(var trackable_object in CheckPointUtils.list_objects(_object_graph_view)) + { + if(trackable_object is TrackableDataStructure && trackable_object._trackable_children().Count == 0) + { + continue; + } + _checkpoint.AllTrackables.Add(trackable_object); + } + var unused_trackables = CheckPointUtils._objects_with_attributes(_checkpoint.AllTrackables) + .Except(_checkpoint.ObjectByProtoId.Values); + if (unused_trackables.Any()) + { + var num_unused_trackables = unused_trackables.Count(); + var num_variables_to_show = Math.Min(10, num_unused_trackables); + throw new AssertionError($"Found {num_unused_trackables} Python objects that were " + + $"not bound to checkpointed values, likely due to changes in the " + + $"Python program. Showing {num_variables_to_show} of " + + $"{num_unused_trackables} unmatched objects: " + + $"{{list(unused_python_objects)[:num_variables_to_show]}}"); + } + return this; + } + + public override LoadStatus assert_nontrivial_match() + { + throw new NotImplementedException(); + } + + public override LoadStatus expect_partial() + { + throw new NotImplementedException(); + } + + public override void initialize_or_restore(Session? session = null) + { + throw new NotImplementedException(); + } + + public override LoadStatus run_restore_ops(Session? session = null) + { + throw new NotImplementedException(); + } } \ No newline at end of file diff --git a/src/TensorFlowNET.Core/Checkpoint/functional_saver.cs b/src/TensorFlowNET.Core/Checkpoint/functional_saver.cs index 09904d684..96e6c8dd9 100644 --- a/src/TensorFlowNET.Core/Checkpoint/functional_saver.cs +++ b/src/TensorFlowNET.Core/Checkpoint/functional_saver.cs @@ -213,7 +213,7 @@ public IDictionary> restore(Tensor file_pref // tf python has code `with ops.device(restore_device):` here. tf.device(restore_device); // may be risky. - var restored_tensors = tf.io.restore_v2(file_prefix, tensor_names.ToArray(), slice_specs.ToArray(), tensor_dtypes.ToArray()); + var restored_tensors = gen_ops.restore_v2(file_prefix, tensor_names.ToArray(), slice_specs.ToArray(), tensor_dtypes.ToArray()); Dictionary> restored_tensor_dict = new(); int idx = 0; diff --git a/src/TensorFlowNET.Core/Checkpoint/restore.cs b/src/TensorFlowNET.Core/Checkpoint/restore.cs new file mode 100644 index 000000000..b27396a79 --- /dev/null +++ b/src/TensorFlowNET.Core/Checkpoint/restore.cs @@ -0,0 +1,331 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using Tensorflow.Train; +using Tensorflow.Training; +using static Tensorflow.Binding; + +namespace Tensorflow.Checkpoint; + +public class CheckpointPosition +{ + private CheckpointRestoreCoordinator _checkpoint; + private int _proto_id; + private bool _skip_restore; + public CheckpointPosition(CheckpointRestoreCoordinator checkpoint, int proto_id) + { + _checkpoint = checkpoint; + _proto_id = proto_id; + _skip_restore = false; + } + + public Trackable Trackable => _checkpoint.ObjectByProtoId[_proto_id]; + public CheckpointRestoreCoordinator Checkpoint => _checkpoint; + public TrackableObjectGraph.Types.TrackableObject ObjectProto => _checkpoint.ObjectGraphProto.Nodes[_proto_id]; + + public void restore(Trackable trackable) + { + using (ops.init_scope()) + { + if (bind_project(trackable)) + { + var restore_ops = _restore_descendants(); + if(restore_ops is not null && restore_ops.Count > 0) + { + _checkpoint.new_restore_ops(restore_ops); + } + } + } + } + + /// + /// Set a checkpoint<->object correspondence. + /// + /// + /// + public bool bind_project(Trackable trackable) + { + _checkpoint.AllTrackables.Add(trackable); + _checkpoint.MatchedProtoIds.Add(_proto_id); + if(_checkpoint.ObjectByProtoId.TryGetValue(_proto_id, out var current_assignment)) + { + // skip the `logging.warning`. + return false; + } + else + { + _checkpoint.ObjectByProtoId[_proto_id] = trackable; + return true; + } + } + + public (List, Dictionary>, List, object?) gather_ops_or_named_saveables() + { + // skip the registered_saver + + if (ObjectProto.Attributes is null || ObjectProto.Attributes.Count == 0) + { + return (new List(), new Dictionary>(), + new List(), null); + } + + var saveable_factories = saveable_object_util.saveable_objects_from_trackable(this.Trackable); + + List existing_restore_ops; + List positions = new(); + Dictionary> named_saveables; + if (saveable_factories.Keys.Count == 1 && saveable_factories.Keys.First() == TrackableUtils.SERIALIZE_TO_TENSORS_NAME) + { + (existing_restore_ops, named_saveables) = _create_serialize_to_tensor_saveable(saveable_factories); + } + else if(saveable_factories.Count > 0) + { + (existing_restore_ops, named_saveables) = _create_saveables_by_attribute_name(saveable_factories); + } + else + { + throw new NotImplementedException(); + } + return (existing_restore_ops, named_saveables, positions, null); + } + + public CheckpointPosition create_child_position(int node_id) + { + return new CheckpointPosition(_checkpoint, node_id); + } + + public (CheckpointPosition, BaseResourceVariable) create_slot_variable_position(Optimizer optimizer_object, BaseResourceVariable variable, + int slot_variable_id, string slot_name) + { + //CheckpointPosition slot_variable_position = new(Checkpoint, slot_variable_id); + + // TODO(Rinne): implement it. + return (null, null); + } + + /// + /// Creates a saveable using the _serialize_to_tensor method. + /// + /// + private (List, Dictionary>) _create_serialize_to_tensor_saveable( + IDictionary>> saveable_factories) + { + string suffix = SaveableCompat.get_saveable_name(this.Trackable); + suffix = suffix ?? ""; + var saveable_name = _extract_saveable_name(ObjectProto.Attributes[0].CheckpointKey) + suffix; + + if (!tf.Context.executing_eagerly()) + { + throw new NotImplementedException("The restore under graph mode has not been implemented. " + + "Please submit an issue to https://github.com/SciSharp/TensorFlow.NET/issues"); + } + + var saveable = saveable_factories[TrackableUtils.SERIALIZE_TO_TENSORS_NAME](saveable_name); + // skip the cache. + Dictionary> dict = new(); + dict[saveable_name] = saveable; + return (new List(), dict); + } + + private (List, Dictionary>) _create_saveables_by_attribute_name( + IDictionary>> saveable_factories) + { + // TODO(Rinne): implement it. + if(ObjectProto.Attributes is null) + { + return (new List(), new Dictionary>()); + } + + List existing_restore_ops = new(); + HashSet created_compat_names = new(); + Dictionary> named_saveables = new(); + foreach (var serialized_tensor in ObjectProto.Attributes) + { + Operation existing_op; + if (tf.Context.executing_eagerly() || !_checkpoint.RestoreOpsByName.ContainsKey(serialized_tensor.CheckpointKey)) + { + existing_op = null; + } + else + { + existing_op = _checkpoint.RestoreOpsByName[serialized_tensor.CheckpointKey]; + } + + if(existing_op is not null) + { + existing_restore_ops.Add(existing_op); + continue; + } + + if(created_compat_names.Any(x => serialized_tensor.Name.StartsWith(x))) + { + continue; + } + + // TODO(Rinne): deal with cache. + + var saveable = _get_saveable_from_factory(saveable_factories, serialized_tensor, created_compat_names); + if(saveable is null) + { + _checkpoint.UnusedAttributes.SetDefault(_proto_id, new List()).Add(serialized_tensor.Name); + continue; + } + named_saveables[serialized_tensor.CheckpointKey] = saveable; + } + return (existing_restore_ops, named_saveables); + } + + private Maybe _get_saveable_from_factory(IDictionary>> saveable_factories, + TrackableObjectGraph.Types.TrackableObject.Types.SerializedTensor serialized_tensor, HashSet created_compat_names) + { + var expected_factory_name = serialized_tensor.Name; + var factory_input_name = serialized_tensor.CheckpointKey; + + if (!saveable_factories.TryGetValue(expected_factory_name, out var matched_factory)) + { + foreach(var item in saveable_factories) + { + var factory_name = item.Key; + var factory = item.Value; + if (expected_factory_name.StartsWith(factory_name)) + { + if(matched_factory is not null) + { + throw new ValueError($"Forward compatibility load error: Unable to load " + + "checkpoint saved in future version of TensorFlow. " + + "Please update your version of TensorFlow to the " + + "version in which the checkpoint was saved."); + } + } + matched_factory = factory; + factory_input_name = _extract_saveable_name(serialized_tensor.CheckpointKey) + factory_name; + created_compat_names.Add(factory_name); + } + } + return matched_factory(factory_input_name); + } + + private string _extract_saveable_name(string checkpoint_key) + { + var search_key = TrackableUtils.OBJECT_ATTRIBUTES_NAME + "/"; + return checkpoint_key.Substring(0, checkpoint_key.IndexOf(search_key) + search_key.Length); + } + + /// + /// Restore the bound Trackable and dependencies (may be deferred). + /// + private List _restore_descendants() + { + Queue<(CheckpointPosition, Trackable)> visit_queue = new(); + visit_queue.Enqueue((this, this.Trackable)); + List restore_ops = new(); + Dictionary> tensor_saveables = new(); + List positions = new(); + + CheckpointPosition current_position = null; + while (visit_queue.Count > 0) + { + current_position = visit_queue.Dequeue().Item1; + var (new_restore_ops, new_tensor_saveables, new_positions, new_registered_savers) = current_position._single_restore(); + restore_ops.AddRange(new_restore_ops); + foreach(var item in new_tensor_saveables) + { + tensor_saveables.Add(item.Key, item.Value); + } + positions.AddRange(new_positions); + _queue_children_for_restoration(current_position, visit_queue); + _queue_slot_variables(current_position, visit_queue); + } + restore_ops.AddRange(current_position.Checkpoint.restore_saveables(tensor_saveables, positions, null)); + return restore_ops; + } + + private void _queue_children_for_restoration(CheckpointPosition checkpoint_position, Queue<(CheckpointPosition, Trackable)> visit_queue) + { + var trackable = checkpoint_position.Trackable; + foreach(var child in checkpoint_position.ObjectProto.Children) + { + var child_position = checkpoint_position.create_child_position(child.NodeId); + var local_object = trackable._lookup_dependency(child.LocalName); + var child_proto = child_position.ObjectProto; + if(local_object is null) + { + if(child_proto.Children.Any() || child_proto.Attributes.Any() || child_proto.SlotVariables.Any()) + { + trackable.DeferredDependencies.SetDefault(child.LocalName, new List()).Add(child_position); + } + } + else + { + if (child_position.bind_project(local_object)) + { + visit_queue.Enqueue((child_position, local_object)); + } + } + } + } + + private void _queue_slot_variables(CheckpointPosition checkpoint_position, Queue<(CheckpointPosition, Trackable)> visit_queue) + { + var trackable = checkpoint_position.Trackable; + var checkpoint = checkpoint_position.Checkpoint; + if(checkpoint.DeferredSlotRestorations.TryGetValue(checkpoint_position._proto_id, out var positions)) + { + checkpoint.DeferredSlotRestorations.Remove(checkpoint_position._proto_id); + foreach (var deferred_slot_restoration in positions) + { + var (slot_variable_position, slot_variable) = checkpoint_position.create_slot_variable_position( + trackable as Optimizer, deferred_slot_restoration.OriginalVariable, deferred_slot_restoration.SlotVariableId, + deferred_slot_restoration.SlotName + ); + if(slot_variable_position is not null) + { + visit_queue.Enqueue((slot_variable_position, slot_variable)); + } + } + } + if (checkpoint.SlotRestorations.TryGetValue(checkpoint_position._proto_id, out var restorations)) + { + checkpoint.SlotRestorations.Remove(checkpoint_position._proto_id); + foreach (var slot_restoration in restorations) + { + if(Checkpoint.ObjectByProtoId.TryGetValue(slot_restoration.OptimizerId, out var optimizer_object)) + { + throw new NotImplementedException(); + // TODO(Rinne)； implement it. + } + else + { + Debug.Assert(trackable is BaseResourceVariable); + Checkpoint.DeferredSlotRestorations.SetDefault(slot_restoration.OptimizerId, new List()) + .Add(new DeferredSlotVariableRestoration(trackable as BaseResourceVariable, slot_restoration.SlotVariableId, slot_restoration.SlotName)); + } + } + } + } + + private (List, Dictionary>, List, object?) _single_restore() + { + var trackable = this.Trackable; + trackable._maybe_initialize_trackable(); + if(_checkpoint.RestoreUid > trackable.UpdateUid) + { + var (restore_ops, tensor_saveables, positions, registered_savers) = gather_ops_or_named_saveables(); + trackable.UpdateUid = _checkpoint.RestoreUid; + return (restore_ops, tensor_saveables, positions, registered_savers); + } + else + { + return (new List(), new Dictionary>(), + new List(), null); + } + } +} + +public record class DeferredSlotVariableRestoration( + BaseResourceVariable OriginalVariable, + int SlotVariableId, + string SlotName +); \ No newline at end of file diff --git a/src/TensorFlowNET.Core/Eager/execute.cs b/src/TensorFlowNET.Core/Eager/execute.cs index cb3ea4d3c..2926f8e28 100644 --- a/src/TensorFlowNET.Core/Eager/execute.cs +++ b/src/TensorFlowNET.Core/Eager/execute.cs @@ -10,7 +10,7 @@ namespace Tensorflow.Eager { - internal class execute + internal static class execute { public static (DataType[], Tensor[]) onvert_to_mixed_eager_tensors(Tensor[] values, Context ctx) { @@ -27,5 +27,9 @@ public static Tensor[] quick_execute(string op_name, int num_outputs, Tensor[] i return tensors; } + public static bool must_record_gradient() + { + return false; + } } } diff --git a/src/TensorFlowNET.Core/Functions/ConcreteFunction.cs b/src/TensorFlowNET.Core/Functions/ConcreteFunction.cs index bac9cedbf..a6720a5f3 100644 --- a/src/TensorFlowNET.Core/Functions/ConcreteFunction.cs +++ b/src/TensorFlowNET.Core/Functions/ConcreteFunction.cs @@ -13,8 +13,8 @@ namespace Tensorflow.Functions /// public class ConcreteFunction: Trackable { - FuncGraph func_graph; - ForwardBackwardCall forward_backward; + internal FuncGraph func_graph; + internal ForwardBackwardCall forward_backward; public Tensor[] Inputs => func_graph.Inputs; public Tensor[] CapturedInputs => func_graph.external_captures; @@ -23,6 +23,8 @@ public class ConcreteFunction: Trackable public Tensor[] Outputs; public Type ReturnType; public TensorSpec[] OutputStructure; + public IEnumerable ArgKeywords { get; set; } + public long NumPositionArgs { get; set; } public ConcreteFunction(string name) { @@ -163,6 +165,15 @@ public Tensors CallFlat(Tensor[] args, Tensor[] captured_inputs) return flat_outputs; } + public void AddTograph(Graph? g = null) + { + if(!tf.Context.executing_eagerly() && g is null) + { + g = ops.get_default_graph(); + } + // TODO(Rinne); complete it with `_delayed_rewrite_functions`. + } + ForwardBackwardCall SelectForwardAndBackwardFunctions(Tensors args, int possible_gradient_type, bool executing_eagerly) { var functions = new FirstOrderTapeGradientFunctions(func_graph, false); diff --git a/src/TensorFlowNET.Core/IO/gfile.cs b/src/TensorFlowNET.Core/IO/gfile.cs index 5f08702da..142b8b64e 100644 --- a/src/TensorFlowNET.Core/IO/gfile.cs +++ b/src/TensorFlowNET.Core/IO/gfile.cs @@ -16,8 +16,10 @@ limitations under the License. using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; +using static Tensorflow.Binding; namespace Tensorflow.IO { @@ -63,5 +65,15 @@ public string[] glob(string data_dir) dirs.AddRange(Directory.GetFiles(dir)); return dirs.ToArray(); } + + public string join(params string[] paths) + { + Debug.Assert(paths.Length >= 1); + if (paths[0].Substring(1).Contains("://")) + { + throw new NotImplementedException("The combination of urls has not been implemented."); + } + return Path.Combine(paths); + } } } diff --git a/src/TensorFlowNET.Core/Keras/Common/CustomizedAxisJsonConverter.cs b/src/TensorFlowNET.Core/Keras/Common/CustomizedAxisJsonConverter.cs index 4e190605c..f6087a43a 100644 --- a/src/TensorFlowNET.Core/Keras/Common/CustomizedAxisJsonConverter.cs +++ b/src/TensorFlowNET.Core/Keras/Common/CustomizedAxisJsonConverter.cs @@ -37,7 +37,16 @@ public override void WriteJson(JsonWriter writer, object? value, JsonSerializer public override object? ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer) { - var axis = serializer.Deserialize(reader, typeof(long[])); + int[]? axis; + if(reader.ValueType == typeof(long)) + { + axis = new int[1]; + axis[0] = (int)serializer.Deserialize(reader, typeof(int)); + } + else + { + axis = serializer.Deserialize(reader, typeof(int[])) as int[]; + } if (axis is null) { throw new ValueError("Cannot deserialize 'null' to `Axis`."); diff --git a/src/TensorFlowNET.Core/Keras/Common/CustomizedDTypeJsonConverter.cs b/src/TensorFlowNET.Core/Keras/Common/CustomizedDTypeJsonConverter.cs new file mode 100644 index 000000000..fce7bec58 --- /dev/null +++ b/src/TensorFlowNET.Core/Keras/Common/CustomizedDTypeJsonConverter.cs @@ -0,0 +1,36 @@ +using Newtonsoft.Json.Linq; +using Newtonsoft.Json; + +namespace Tensorflow.Keras.Common +{ + public class CustomizedDTypeJsonConverter : JsonConverter + { + public override bool CanConvert(Type objectType) + { + return objectType == typeof(TF_DataType); + } + + public override bool CanRead => true; + + public override bool CanWrite => true; + + public override void WriteJson(JsonWriter writer, object? value, JsonSerializer serializer) + { + var token = JToken.FromObject(dtypes.as_numpy_name((TF_DataType)value)); + token.WriteTo(writer); + } + + public override object? ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer) + { + if (reader.ValueType == typeof(string)) + { + var str = (string)serializer.Deserialize(reader, typeof(string)); + return dtypes.tf_dtype_from_name(str); + } + else + { + return (TF_DataType)serializer.Deserialize(reader, typeof(int)); + } + } + } +} diff --git a/src/TensorFlowNET.Core/Keras/Common/CustomizedNodeConfigJsonConverter.cs b/src/TensorFlowNET.Core/Keras/Common/CustomizedNodeConfigJsonConverter.cs index 1ad19fc89..cfd8ee8f7 100644 --- a/src/TensorFlowNET.Core/Keras/Common/CustomizedNodeConfigJsonConverter.cs +++ b/src/TensorFlowNET.Core/Keras/Common/CustomizedNodeConfigJsonConverter.cs @@ -46,7 +46,16 @@ public override void WriteJson(JsonWriter writer, object? value, JsonSerializer { throw new ValueError("Cannot deserialize 'null' to `Shape`."); } - if(values.Length != 3) + if(values.Length == 1) + { + var array = values[0] as JArray; + if(array is null) + { + throw new ValueError($"The value ({string.Join(", ", values)}) cannot be deserialized to type `NodeConfig`."); + } + values = array.ToObject(); + } + if (values.Length < 3) { throw new ValueError($"The value ({string.Join(", ", values)}) cannot be deserialized to type `NodeConfig`."); } @@ -54,19 +63,37 @@ public override void WriteJson(JsonWriter writer, object? value, JsonSerializer { throw new TypeError($"The first value of `NodeConfig` is expected to be `string`, but got `{values[0].GetType().Name}`"); } - if (values[1] is not int) + int nodeIndex; + int tensorIndex; + if (values[1] is long) + { + nodeIndex = (int)(long)values[1]; + } + else if (values[1] is int) + { + nodeIndex = (int)values[1]; + } + else { throw new TypeError($"The first value of `NodeConfig` is expected to be `int`, but got `{values[1].GetType().Name}`"); } - if (values[2] is not int) + if (values[2] is long) + { + tensorIndex = (int)(long)values[2]; + } + else if (values[1] is int) + { + tensorIndex = (int)values[2]; + } + else { throw new TypeError($"The first value of `NodeConfig` is expected to be `int`, but got `{values[2].GetType().Name}`"); } return new NodeConfig() { Name = values[0] as string, - NodeIndex = (int)values[1], - TensorIndex = (int)values[2] + NodeIndex = nodeIndex, + TensorIndex = tensorIndex }; } } diff --git a/src/TensorFlowNET.Core/Keras/Common/CustomizedShapeJsonConverter.cs b/src/TensorFlowNET.Core/Keras/Common/CustomizedShapeJsonConverter.cs index 300cb2f28..198662afe 100644 --- a/src/TensorFlowNET.Core/Keras/Common/CustomizedShapeJsonConverter.cs +++ b/src/TensorFlowNET.Core/Keras/Common/CustomizedShapeJsonConverter.cs @@ -51,10 +51,28 @@ public override void WriteJson(JsonWriter writer, object? value, JsonSerializer public override object? ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer) { - var dims = serializer.Deserialize(reader, typeof(long?[])) as long?[]; - if(dims is null) + long?[] dims; + try { - throw new ValueError("Cannot deserialize 'null' to `Shape`."); + dims = serializer.Deserialize(reader, typeof(long?[])) as long?[]; + } + catch (JsonSerializationException ex) + { + if (reader.Value.Equals("class_name")) + { + reader.Read(); + reader.Read(); + reader.Read(); + dims = serializer.Deserialize(reader, typeof(long?[])) as long?[]; + } + else + { + throw ex; + } + } + if (dims is null) + { + return null; } long[] convertedDims = new long[dims.Length]; for(int i = 0; i < dims.Length; i++) diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs index 036291076..20a98e3d3 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayer.cs @@ -19,6 +19,7 @@ public interface ILayer: IWithTrackable, IKerasConfigable List TrainableVariables { get; } List TrainableWeights { get; } List NonTrainableWeights { get; } + List Weights { get; } Shape OutputShape { get; } Shape BatchInputShape { get; } TensorShapeConfig BuildInputShape { get; } diff --git a/src/TensorFlowNET.Core/Keras/Saving/ModelConfig.cs b/src/TensorFlowNET.Core/Keras/Saving/ModelConfig.cs index cac19180f..934d3b151 100644 --- a/src/TensorFlowNET.Core/Keras/Saving/ModelConfig.cs +++ b/src/TensorFlowNET.Core/Keras/Saving/ModelConfig.cs @@ -1,8 +1,11 @@ using Newtonsoft.Json; +using Newtonsoft.Json.Linq; using System; using System.Collections.Generic; using System.Text; +using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; +using static Google.Protobuf.Reflection.FieldDescriptorProto.Types; namespace Tensorflow.Keras.Saving { diff --git a/src/TensorFlowNET.Core/ModelSaving/ModelSaver.cs b/src/TensorFlowNET.Core/ModelSaving/ModelSaver.cs index 4437ba0aa..9ff381299 100644 --- a/src/TensorFlowNET.Core/ModelSaving/ModelSaver.cs +++ b/src/TensorFlowNET.Core/ModelSaving/ModelSaver.cs @@ -3,6 +3,7 @@ using System.Text; using Tensorflow.Keras.Engine; using Tensorflow.Train; +using Tensorflow.Training.Saving.SavedModel; namespace Tensorflow.ModelSaving { diff --git a/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs b/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs index 2b83dd1d1..4e9369a8b 100644 --- a/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs +++ b/src/TensorFlowNET.Core/Operations/NnOps/RNNCell.cs @@ -71,6 +71,7 @@ public abstract class RnnCell : ILayer, RNNArgs.IRnnArgCell public List TrainableVariables => throw new NotImplementedException(); public List TrainableWeights => throw new NotImplementedException(); + public List Weights => throw new NotImplementedException(); public List NonTrainableWeights => throw new NotImplementedException(); public Shape OutputShape => throw new NotImplementedException(); diff --git a/src/TensorFlowNET.Core/Operations/gen_ops.cs b/src/TensorFlowNET.Core/Operations/gen_ops.cs index 956be96b5..26a9b5be8 100644 --- a/src/TensorFlowNET.Core/Operations/gen_ops.cs +++ b/src/TensorFlowNET.Core/Operations/gen_ops.cs @@ -27189,8 +27189,33 @@ public static Tensor restore_slice(Tensor file_pattern, Tensor tensor_name, Tens /// /// Callers must ensure all the named tensors are indeed stored in the checkpoint. /// - public static Tensor[] restore_v2(Tensor prefix, Tensor tensor_names, Tensor shape_and_slices, TF_DataType[] dtypes, string name = "RestoreV2") + public static Tensor[] restore_v2(Tensor prefix, string[] tensor_names, string[] shape_and_slices, TF_DataType[] dtypes, string name = "RestoreV2") { + var ctx = tf.Context; + if (ctx.executing_eagerly()) + { + try + { + Dictionary attrs = new(); + attrs["dtypes"] = dtypes; + var result = tf.Runner.TFE_FastPathExecute(new FastPathOpExecInfo( + "RestoreV2", name, prefix, tensor_names, shape_and_slices + ) + { attrs = attrs }); + return result; + } + catch (Exception) + { + try + { + return restore_v2_eager_fallback(prefix, tensor_names, shape_and_slices, dtypes, name, ctx); + } + catch (Exception) + { + + } + } + } var dict = new Dictionary(); dict["prefix"] = prefix; dict["tensor_names"] = tensor_names; @@ -27202,6 +27227,22 @@ public static Tensor[] restore_v2(Tensor prefix, Tensor tensor_names, Tensor sha return (tensors); } + public static Tensor[] restore_v2_eager_fallback(Tensor prefix, string[] tensor_names, string[] shape_and_slices, TF_DataType[] dtypes, string name, Context ctx) + { + prefix = ops.convert_to_tensor(prefix, TF_DataType.TF_STRING); + var tensor_names_tensor = ops.convert_to_tensor(tensor_names, TF_DataType.TF_STRING); + var shape_and_slices_tensor = ops.convert_to_tensor(shape_and_slices, TF_DataType.TF_STRING); + object[] attrs = new object[] { "dtypes", dtypes }; + Tensor[] inputs_flat = new Tensor[] { prefix, tensor_names_tensor, shape_and_slices_tensor }; + var result = execute.quick_execute("RestoreV2", dtypes.Length, inputs_flat, attrs, ctx, name); + + if (execute.must_record_gradient()) + { + // TODO(Rinne); record the gradient + } + return result; + } + /// /// Reverses specific dimensions of a tensor. /// diff --git a/src/TensorFlowNET.Core/Operations/io_ops.cs b/src/TensorFlowNET.Core/Operations/io_ops.cs index 35c5877f3..16e1bac47 100644 --- a/src/TensorFlowNET.Core/Operations/io_ops.cs +++ b/src/TensorFlowNET.Core/Operations/io_ops.cs @@ -62,6 +62,7 @@ public Operation save_v2_eager_fallback(Tensor prefix, string[] tensor_names, st public Tensor[] restore_v2(Tensor prefix, string[] tensor_names, string[] shape_and_slices, TF_DataType[] dtypes, string name = null) { + // Note: this implementation is not correct in many cases, please consider using `gen_ops.restore_v2`. var _op = tf.OpDefLib._apply_op_helper("RestoreV2", name: name, args: new { prefix, tensor_names, shape_and_slices, dtypes }); return _op.outputs; diff --git a/src/TensorFlowNET.Core/Operations/resource_variable_ops.cs b/src/TensorFlowNET.Core/Operations/resource_variable_ops.cs index 1b1fa0037..6ce7a0b00 100644 --- a/src/TensorFlowNET.Core/Operations/resource_variable_ops.cs +++ b/src/TensorFlowNET.Core/Operations/resource_variable_ops.cs @@ -17,8 +17,8 @@ limitations under the License. using System; using System.Linq; using Tensorflow.Framework; -using Tensorflow.ModelSaving; using Tensorflow.Train; +using Tensorflow.Training.Saving.SavedModel; using Tensorflow.Variables; using static Tensorflow.CppShapeInferenceResult.Types; diff --git a/src/TensorFlowNET.Core/Tensors/TF_DataType.cs b/src/TensorFlowNET.Core/Tensors/TF_DataType.cs index 5fe28c5d1..0f514b429 100644 --- a/src/TensorFlowNET.Core/Tensors/TF_DataType.cs +++ b/src/TensorFlowNET.Core/Tensors/TF_DataType.cs @@ -1,9 +1,13 @@ -namespace Tensorflow +using Newtonsoft.Json; +using Tensorflow.Keras.Common; + +namespace Tensorflow { /// /// TF_DataType holds the type for a scalar value. E.g., one slot in a tensor. /// The enum values here are identical to corresponding values in types.proto. /// + [JsonConverter(typeof(CustomizedDTypeJsonConverter))] public enum TF_DataType { DtInvalid = 0, diff --git a/src/TensorFlowNET.Core/Tensors/dtypes.cs b/src/TensorFlowNET.Core/Tensors/dtypes.cs index deeb9e4b5..3563f91a0 100644 --- a/src/TensorFlowNET.Core/Tensors/dtypes.cs +++ b/src/TensorFlowNET.Core/Tensors/dtypes.cs @@ -159,7 +159,10 @@ public static TF_DataType tf_dtype_from_name(string name) "uint32" => TF_DataType.TF_UINT32, "int64" => TF_DataType.TF_INT64, "uint64" => TF_DataType.TF_UINT64, + "float16" => TF_DataType.TF_BFLOAT16, + "float32" => TF_DataType.TF_FLOAT, "single" => TF_DataType.TF_FLOAT, + "float64" => TF_DataType.TF_DOUBLE, "double" => TF_DataType.TF_DOUBLE, "complex" => TF_DataType.TF_COMPLEX128, "string" => TF_DataType.TF_STRING, diff --git a/src/TensorFlowNET.Core/Training/Saving/SaveableObject.cs b/src/TensorFlowNET.Core/Training/Saving/SaveableObject.cs index 1309a6174..2fd0d1d83 100644 --- a/src/TensorFlowNET.Core/Training/Saving/SaveableObject.cs +++ b/src/TensorFlowNET.Core/Training/Saving/SaveableObject.cs @@ -39,6 +39,24 @@ public Tensor op _op = value; } } + public BaseResourceVariable variable + { + get + { + if (_op.TryGet(out var v)) + { + return v; + } + else + { + throw new TypeError("The _op is not a variable."); + } + } + set + { + _op = value; + } + } public SaveSpec[] specs; public string name; public string device; diff --git a/src/TensorFlowNET.Core/Training/Saving/SavedModel/LoadOptions.cs b/src/TensorFlowNET.Core/Training/Saving/SavedModel/LoadOptions.cs new file mode 100644 index 000000000..df9bdc1b5 --- /dev/null +++ b/src/TensorFlowNET.Core/Training/Saving/SavedModel/LoadOptions.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Tensorflow +{ + public record class LoadOptions + { + public bool allow_partial_checkpoint; + public string experimental_io_device; + public bool experimental_skip_checkpoint; + public VariablePolicy experimental_variable_policy; + + public LoadOptions(bool allow_partial_checkpoint = false, string experimental_io_device = null, + bool experimental_skip_checkpoint = false, string experimental_variable_policy = null) + { + this.allow_partial_checkpoint = allow_partial_checkpoint; + this.experimental_io_device = experimental_io_device; + this.experimental_skip_checkpoint = experimental_skip_checkpoint; + this.experimental_variable_policy = VariablePolicy.from_obj(experimental_variable_policy); + } + } +} diff --git a/src/TensorFlowNET.Core/Training/Saving/SavedModel/RevivedTypes.cs b/src/TensorFlowNET.Core/Training/Saving/SavedModel/RevivedTypes.cs index fe0403c30..601882930 100644 --- a/src/TensorFlowNET.Core/Training/Saving/SavedModel/RevivedTypes.cs +++ b/src/TensorFlowNET.Core/Training/Saving/SavedModel/RevivedTypes.cs @@ -1,4 +1,5 @@ -using Tensorflow.Train; +using System; +using Tensorflow.Train; namespace Tensorflow; @@ -14,4 +15,10 @@ public class RevivedTypes // TODO: complete the implementation. return null; } + + public static Tuple> deserialize(object proto) + { + // TODO: complete the implementation. + return null; + } } diff --git a/src/TensorFlowNET.Core/ModelSaving/SaveOptions.cs b/src/TensorFlowNET.Core/Training/Saving/SavedModel/SaveOptions.cs similarity index 83% rename from src/TensorFlowNET.Core/ModelSaving/SaveOptions.cs rename to src/TensorFlowNET.Core/Training/Saving/SavedModel/SaveOptions.cs index 45ebd884f..d42f52535 100644 --- a/src/TensorFlowNET.Core/ModelSaving/SaveOptions.cs +++ b/src/TensorFlowNET.Core/Training/Saving/SavedModel/SaveOptions.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using System.Text; -namespace Tensorflow.ModelSaving +namespace Tensorflow { /// /// Options for saving to SavedModel. @@ -35,7 +35,7 @@ private VariablePolicy(string policy) public bool save_variable_devices() { - return this != VariablePolicy.None; + return this != None; } /// @@ -45,14 +45,14 @@ public bool save_variable_devices() /// public static VariablePolicy from_obj(object obj) { - if (obj is null) return VariablePolicy.None; + if (obj is null) return None; if (obj is VariablePolicy) return (VariablePolicy)obj; var key = obj.ToString().ToLower(); return key switch { - null => VariablePolicy.None, - "save_variable_devices" => VariablePolicy.SAVE_VARIABLE_DEVICES, - "expand_distributed_variables" => VariablePolicy.EXPAND_DISTRIBUTED_VARIABLES, + null => None, + "save_variable_devices" => SAVE_VARIABLE_DEVICES, + "expand_distributed_variables" => EXPAND_DISTRIBUTED_VARIABLES, _ => throw new ValueError($"Received invalid VariablePolicy value: {obj}.") }; } diff --git a/src/TensorFlowNET.Core/Training/Saving/SavedModel/SaveableView.cs b/src/TensorFlowNET.Core/Training/Saving/SavedModel/SaveableView.cs index 1be54287e..5752d7284 100644 --- a/src/TensorFlowNET.Core/Training/Saving/SavedModel/SaveableView.cs +++ b/src/TensorFlowNET.Core/Training/Saving/SavedModel/SaveableView.cs @@ -5,7 +5,6 @@ using Tensorflow.Checkpoint; using Tensorflow.Contexts; using Tensorflow.Functions; -using Tensorflow.ModelSaving; using Tensorflow.Train; using Tensorflow.Training; using pbc = global::Google.Protobuf.Collections; diff --git a/src/TensorFlowNET.Core/Training/Saving/SavedModel/WrapperFunction.cs b/src/TensorFlowNET.Core/Training/Saving/SavedModel/WrapperFunction.cs new file mode 100644 index 000000000..341a12ab9 --- /dev/null +++ b/src/TensorFlowNET.Core/Training/Saving/SavedModel/WrapperFunction.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Tensorflow.Functions; + +namespace Tensorflow.Training.Saving.SavedModel +{ + /// + /// A class wraps a concrete function to handle different distributed contexts. + /// + internal class WrapperFunction: ConcreteFunction + { + public WrapperFunction(ConcreteFunction concrete_function): base(concrete_function.func_graph) + { + this.forward_backward = concrete_function.forward_backward; + this.Outputs = concrete_function.Outputs; + this.ReturnType = concrete_function.ReturnType; + this.OutputStructure = concrete_function.OutputStructure; + this.ArgKeywords = concrete_function.ArgKeywords; + } + } +} diff --git a/src/TensorFlowNET.Core/Training/Saving/SavedModel/function_deserialization.cs b/src/TensorFlowNET.Core/Training/Saving/SavedModel/function_deserialization.cs new file mode 100644 index 000000000..5b482872d --- /dev/null +++ b/src/TensorFlowNET.Core/Training/Saving/SavedModel/function_deserialization.cs @@ -0,0 +1,36 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Tensorflow.Functions; +using Tensorflow.Util; + +namespace Tensorflow.Training.Saving.SavedModel +{ + public static class function_deserialization + { + public static ConcreteFunction setup_bare_concrete_function(SavedBareConcreteFunction saved_bare_concrete_function, + IDictionary concrete_functions) + { + var concrete_function = concrete_functions[saved_bare_concrete_function.ConcreteFunctionName]; + concrete_function.ArgKeywords = saved_bare_concrete_function.ArgumentKeywords.ToList(); + concrete_function.NumPositionArgs = saved_bare_concrete_function.AllowedPositionalArguments; + + var function_spec = _deserialize_function_spec_as_nonmethod(saved_bare_concrete_function.FunctionSpec); + concrete_function.AddTograph(); + return concrete_function; + } + + private static FunctionSpec _deserialize_function_spec_as_nonmethod(FunctionSpec function_spec_proto) + { + // TODO(Rinne)； revise the implementation. + return new FunctionSpec() + { + Fullargspec = function_spec_proto.Fullargspec, + IsMethod = function_spec_proto.IsMethod, + InputSignature = function_spec_proto.InputSignature, + JitCompile = function_spec_proto.JitCompile + }; + } + } +} diff --git a/src/TensorFlowNET.Core/Training/Saving/SavedModel/loader.cs b/src/TensorFlowNET.Core/Training/Saving/SavedModel/loader.cs new file mode 100644 index 000000000..da999b376 --- /dev/null +++ b/src/TensorFlowNET.Core/Training/Saving/SavedModel/loader.cs @@ -0,0 +1,641 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Net.Sockets; +using System.Text; +using Tensorflow.Checkpoint; +using Tensorflow.Train; +using Tensorflow.Training; +using pbc = global::Google.Protobuf.Collections; +using static Tensorflow.Binding; +using System.Runtime.CompilerServices; +using Tensorflow.Variables; +using Tensorflow.Functions; +using Tensorflow.Training.Saving.SavedModel; + +namespace Tensorflow +{ + /// + /// Helper class to load an object-based SavedModel. + /// + public partial class Loader + { + private pbc::RepeatedField _asset_file_def; + private Dictionary> _operation_attributes; + private SavedObjectGraph _proto; + private string _export_dir; + private CheckpointOptions _checkpoint_options; + private LoadOptions _save_options; + private IDictionary)> _node_filters; + private Dictionary? _node_path_to_id; + private List? _filtered_nodes; + private List _ordered_node_ids; + private Dictionary)> _loaded_nodes; + private List _nodes; + private Dictionary> _node_setters; + public Loader(SavedObjectGraph object_graph_proto, SavedModel saved_model_proto, string export_dir, + CheckpointOptions ckpt_options, LoadOptions save_options, IDictionary)> filters) + { + var meta_graph = saved_model_proto.MetaGraphs[0]; + _asset_file_def = meta_graph.AssetFileDef; + _operation_attributes = meta_graph.GraphDef.Node.ToDictionary(x => x.Name, x => x.Attr); + _proto = object_graph_proto; + _export_dir = export_dir; + // TODO: `this._concrete_functions` and `this._restored_concrete_functions` + _checkpoint_options = ckpt_options; + _save_options = save_options; + + // TODO: `this._pretty_printer` + + _node_filters = filters; + _node_path_to_id = _convert_node_paths_to_ints(); + _loaded_nodes = new Dictionary)>(); + foreach(var filter in filters) + { + _loaded_nodes[_node_path_to_id[filter.Key]] = filter.Value; + } + + _filtered_nodes = _retrieve_all_filtered_nodes(); + + _ordered_node_ids = _generate_ordered_node_ids(); + + _load_all(); + + + if (!save_options.experimental_skip_checkpoint) + { + _restore_checkpoint(); + } + foreach(var node in _nodes) + { + // skip the process of `CapturableResource`. + } + } + + /// + /// Maps all string node paths in node_filters to the int node ids. + /// + /// + private Dictionary? _convert_node_paths_to_ints() + { + if( _node_filters is null) + { + return null; + } + Dictionary path_to_int = new(); + foreach(var node_id in _node_filters.Keys) + { + int int_node_id; + var node_path = node_id.Split('.'); + if (node_path[0] != "root") + { + throw new ValueError($"When passing string identifiers to node_filters, the first name" + + $" must be root. Received {node_path[0]}."); + } + int_node_id = 0; + for(int i = 0; i < node_path.Length - 1; i++) + { + var name = node_path[i + 1]; + int_node_id = _find_node_child(int_node_id, name, String.Join(".", node_path.Take(i + 1))); + } + path_to_int[node_id] = int_node_id; + } + return path_to_int; + } + + private int _find_node_child(int node_id, string child_name, string path) + { + foreach(var refer in _proto.Nodes[node_id].Children) + { + if(refer.LocalName == child_name) + { + return refer.NodeId; + } + } + throw new ValueError($"Unable to find node {path}."); + } + + private List? _retrieve_all_filtered_nodes() + { + if(_node_filters is null) + { + return null; + } + + HashSet all_filtered_nodes = new(); + Queue nodes_to_visit = new Queue(_node_filters.Keys); + + while(nodes_to_visit.Count > 0) + { + var node_path = nodes_to_visit.Dequeue(); + var node_id = _node_path_to_id[node_path]; + if (all_filtered_nodes.Contains(node_id)) + { + continue; + } + all_filtered_nodes.Add(node_id); + Trackable node = null; + Action setter = null; + if(_loaded_nodes.TryGetValue(node_id, out var res)) + { + (node, setter) = res; + } + if(node is not null) + { + node._maybe_initialize_trackable(); + } + + foreach(var refer in _proto.Nodes[node_id].Children) + { + Trackable children_object = null; + if(_loaded_nodes.TryGetValue(refer.NodeId, out var result)) + { + children_object = result.Item1; + } + // See if node already tracks the child reference, in which case add the child to the loaded_nodes dict. + if(children_object is null && node is not null) + { + children_object = node._lookup_dependency(refer.LocalName); + if(children_object is TrackableDataStructure) + { + // TODO: set setter as lambda. + + _loaded_nodes[refer.NodeId] = (children_object, setter); + } + } + string child_path = $"{node_path}.{refer.LocalName}"; + _node_path_to_id[child_path] = refer.NodeId; + nodes_to_visit.Enqueue(child_path); + } + } + + if (all_filtered_nodes.Contains(0)) + { + return null; + } + return all_filtered_nodes.ToList(); + } + + /// + /// Orders the node ids so that dependencies appear first. + /// + /// + private List _generate_ordered_node_ids() + { + List unordered_ids; + if(_filtered_nodes is null) + { + unordered_ids = Enumerable.Range(0, _proto.Nodes.Count).ToList(); + } + else + { + unordered_ids = new List(_filtered_nodes); + } + + Dictionary> dependency_map = new(); + foreach(var node_id in unordered_ids) + { + var deps = dependency_map.SetDefault(node_id, new List()); + if (_loaded_nodes.ContainsKey(node_id)) + { + continue; + } + var proto = _proto.Nodes[node_id]; + foreach(var dep in _get_node_dependencies(proto).Values.Distinct()) + { + deps.Add(dep); + if(_filtered_nodes is not null && !_filtered_nodes.Contains(dep)) + { + // TODO: add info with `_pretty_printer`. + throw new ValueError($"Unable to partially load SavedModel since the specified filter " + + $"does not include all required objects for loading (e.g. " + + $"variables used in functions or deserialization dependencies). " + + $"Please include this path in the filter: {dep}"); + } + } + int? prev_slot = null; + foreach(var slot_variable_proto in proto.SlotVariables) + { + var slot_variable_node_id = slot_variable_proto.SlotVariableNodeId; + // The optimizer and original variable must be created before the slot + // variable, since the slot variable is generated using the Optimizer's + // add_slot API. + var slot_deps = dependency_map[slot_variable_node_id]; + slot_deps.Add(node_id); + slot_deps.Add(slot_variable_proto.OriginalVariableNodeId); + + if(prev_slot is not null) + { + slot_deps.Add(prev_slot.Value); + } + prev_slot = slot_variable_node_id; + } + } + try + { + return TrackableUtils.order_by_dependency(dependency_map.ToDictionary(x => x.Key, x => x.Value as IEnumerable)); + } + catch (TrackableUtils.CyclicDependencyError ex) + { + throw new ValueError("Encountered a cycle in the deserialization dependencies" + + "in the SavedModel. This is extremely unexpected, please" + + "file a bug and make sure you are not manually modifying the SavedModel."); + } + } + + /// + /// Returns a dictionary of all dependencies of an object. + /// + /// + /// + private Dictionary, int> _get_node_dependencies(SavedObject proto) + { + Dictionary, int> dependencies = new(); + foreach(var refer in proto.Dependencies) + { + dependencies[refer.LocalName] = refer.NodeId; + } + if(proto.KindCase == SavedObject.KindOneofCase.Function) + { + var concreete_functions = proto.Function.ConcreteFunctions; + foreach(var fn_name in concreete_functions) + { + foreach(var bound_input in _proto.ConcreteFunctions[fn_name].BoundInputs) + { + dependencies[bound_input] = bound_input; + } + } + } + else if(proto.KindCase == SavedObject.KindOneofCase.BareConcreteFunction) + { + var fn_name = proto.BareConcreteFunction.ConcreteFunctionName; + foreach(var bound_input in _proto.ConcreteFunctions[fn_name].BoundInputs) + { + dependencies[bound_input] = bound_input; + } + } + else if(proto.KindCase == SavedObject.KindOneofCase.Resource) + { + foreach(var child in proto.Children) + { + if(child.LocalName == "_create_resource") + { + dependencies["_create_resource"] = child.NodeId; + } + } + } + return dependencies; + } + + /// + /// Loads all nodes and functions from the SavedModel and their edges. + /// + private void _load_all() + { + _load_nodes(); + _load_edges(); + + _setup_remaining_functions(); + _load_checkpoint_save_and_restore_functions(); + } + + /// + /// Restores the checkpoint-related save/restore functions to all nodes. + /// + private void _load_checkpoint_save_and_restore_functions() + { + foreach(var (node_id, proto) in _iter_all_nodes()) + { + var node = get(node_id); + if(node is null) + { + // skip it because now we skip the restoration of `Function` and `ConcreteFunction`. + continue; + } + if(proto.SaveableObjects.Keys.Count == 1 && proto.SaveableObjects.First().Key == TrackableUtils.SERIALIZE_TO_TENSORS_NAME) + { + // Restore Trackable serialize- and restore-from-tensor functions. + Debug.Assert(proto.SaveableObjects.Count == 1); + var saveable_object_proto = proto.SaveableObjects.Values.First(); + var save_fn_id = saveable_object_proto.SaveFunction; + var restore_fn_id = saveable_object_proto.RestoreFunction; + + throw new NotImplementedException("Not implemented, please submit an issue to https://github.com/SciSharp/TensorFlow.NET/issues"); + } + else + { + // Restore legacy SaveableObject functions. + Dictionary saveable_fn_by_name = new(); + foreach(var item in proto.SaveableObjects) + { + var name = item.Key; + var saveable_object_proto = item.Value; + var save_fn_id = saveable_object_proto.SaveFunction; + var restore_fn_id = saveable_object_proto.RestoreFunction; + saveable_fn_by_name[name] = (get(save_fn_id), get(restore_fn_id)); + } + node.SelfSaveableObjectFactories = saveable_object_util.recreate_saveable_objects(saveable_fn_by_name, null); + } + } + } + + /// + /// Load all saved objects. + /// + private void _load_nodes() + { + // `nodes` maps from node ids to recreated objects + // `node_setters` maps from node ids to setter functions + // (same signature as setattr) for setting children. + var (nodes, node_setters) = _initialize_loaded_nodes(); + + Dictionary + slot_variable_node_ids = new(); + + foreach(var (node_id, proto) in _iter_all_nodes()) + { + foreach(var slot_variable_proto in proto.SlotVariables) + { + var slot_variable_node_id = slot_variable_proto.SlotVariableNodeId; + slot_variable_node_ids[slot_variable_node_id] = (node_id, slot_variable_proto); + } + } + + // Re-create everything. + foreach (var (node_id, proto) in _iter_all_nodes()) + { + if (nodes.ContainsKey(node_id)) + { + continue; + } + else if (slot_variable_node_ids.ContainsKey(node_id)) + { + // Use the public Optimizer interface when creating slot variables. + var (optimizer_node_id, slot_variable_proto) = slot_variable_node_ids[node_id]; + var optimizer_object = nodes[optimizer_node_id]; + var optimizer_variable = nodes[slot_variable_proto.OriginalVariableNodeId]; + + // TODO: implement it. + throw new NotImplementedException("The model loading of SavedModel still has some incompleted part." + + " Please submit an issue to https://github.com/SciSharp/TensorFlow.NET/issues."); + } + else + { + // skip the function and concrete function. + if(proto.KindCase == SavedObject.KindOneofCase.BareConcreteFunction || proto.KindCase == SavedObject.KindOneofCase.Function) + { + nodes[node_id] = null; + node_setters[node_id] = null; + continue; + } + var (node, setter) = _recreate(proto, node_id, nodes); + nodes[node_id] = node; + node_setters[node_id] = setter; + } + } + + if (!nodes.ContainsKey(0)) + { + nodes[0] = _recreate_base_user_object().Item1; + } + _nodes = new List(); + for(int i = 0; i < _proto.Nodes.Count; i++) + { + _nodes.Add(nodes[i]); + } + _node_setters = node_setters; + } + + /// + /// Load state from checkpoint into the deserialized objects. + /// + private void _restore_checkpoint() + { + var variables_path = SavedModelUtils.get_variables_path(_export_dir); + var saver = new TrackableSaver(new ObjectGraphView(get(0))); + tf.device("CPU"); + saver.FilePrefixPlaceHolder = constant_op.constant(variables_path); + LoadStatus load_status; + if (_save_options.allow_partial_checkpoint) + { + load_status = saver.restore(variables_path, _checkpoint_options).expect_partial(); + load_status.assert_nontrivial_match(); + } + else + { + load_status = saver.restore(variables_path, _checkpoint_options); + load_status.assert_existing_objects_matched(); + } + var ckpt = (load_status as CheckpointLoadStatus).Checkpoint; + + if (!tf.Context.executing_eagerly()) + { + throw new NotImplementedException("The checkpoint restore has not supported graph mode. " + + "Please submit an issue to https://github.com/SciSharp/TensorFlow.NET/issues"); + } + } + + /// + /// Adds edges from objects to other objects and functions. + /// + private void _load_edges() + { + foreach(var (node_id, object_proto) in _iter_all_nodes()) + { + _add_object_graph_edges(object_proto, node_id); + } + + if(_filtered_nodes is not null && _filtered_nodes.Contains(0)) + { + var root = get(0); + foreach(var node_path in _node_filters.Keys) + { + var loaded_node = _nodes[_node_path_to_id[node_path]]; + + var path = node_path.Split('.'); + var current_node = root; + foreach(var name in path.Skip(1).Take(path.Length - 2)) + { + // `hasattr` and `setattr` is used here + throw new NotImplementedException(); + } + // `hasattr` and `setattr` is used here + throw new NotImplementedException(); + } + } + } + + private void _setup_remaining_functions() + { + // TODO: implement it with concrete functions. + } + + public Trackable get(int node_id) + { + return _nodes[node_id]; + } + + public Trackable get(string node_id) + { + return get(_node_path_to_id[node_id]); + } + + /// + /// Adds edges from an object to its children. + /// + /// + /// + private void _add_object_graph_edges(SavedObject proto, int node_id) + { + var obj = _nodes[node_id]; + var setter = _node_setters[node_id]; + + foreach(var refer in proto.Children) + { + if(obj is null) + { + // skip it because now we skip the restoration of `Function` and `ConcreteFunction`. + continue; + } + setter.Invoke(obj, refer.LocalName, _nodes[refer.NodeId]); + // skip the process of "__call__" + } + } + + private (Dictionary, Dictionary>) _initialize_loaded_nodes() + { + Dictionary nodes = new(); + Dictionary> node_setters = new(); + foreach(var item in _loaded_nodes) + { + var node_id = item.Key; + var (node, setter) = item.Value; + nodes[node_id] = node; + node_setters[node_id] = setter; + } + return (nodes, node_setters); + } + + private IEnumerable<(int, SavedObject)> _iter_all_nodes() + { + foreach(var node_id in _ordered_node_ids) + { + yield return (node_id, _proto.Nodes[node_id]); + } + } + + private (Trackable, Action) _recreate(SavedObject proto, int node_id, IDictionary nodes) + { + // skip the registered classes. + + Dictionary, Trackable> dependencies = new(); + foreach(var item in _get_node_dependencies(proto)) + { + dependencies[item.Key] = nodes[item.Value]; + } + + return _recreate_default(proto, node_id, dependencies); + } + + /// + /// Creates a Python object from a SavedObject protocol buffer. + /// + /// + /// + /// + private (Trackable, Action) _recreate_default(SavedObject proto, int node_id, IDictionary, Trackable> dependencies) + { + return proto.KindCase switch + { + SavedObject.KindOneofCase.UserObject => _recreate_user_object(proto.UserObject, node_id), + SavedObject.KindOneofCase.Function => throw new NotImplementedException(), + SavedObject.KindOneofCase.BareConcreteFunction => throw new NotImplementedException(), + SavedObject.KindOneofCase.Variable => _recreate_variable(proto.Variable), + SavedObject.KindOneofCase.CapturedTensor => throw new NotImplementedException() + }; + } + + private (Trackable, Action) _recreate_user_object(SavedUserObject? proto, int node_id) + { + // skip the check of proto identifier because of lack of property. + + var looked_up = RevivedTypes.deserialize(proto); + if(looked_up is null) + { + return _recreate_base_user_object(proto, node_id); + } + return (looked_up.Item1, looked_up.Item2); + } + + private (Trackable, Action) _recreate_base_user_object(SavedUserObject? proto = null, int? node_id = null) + { + return (new _UserObject(), setattr); + } + + private (BaseResourceVariable, Action) _recreate_variable(SavedVariable proto) + { + string name = proto.Name; + string dbg_name = !string.IsNullOrEmpty(name) ? name : ""; + + // TODO(Rinne): `validate_synchronization_aggregation_trainable` + + var (synchronization, aggregation, trainable) = ResourceVariable.validate_synchronization_aggregation_trainable( + proto.Synchronization, proto.Aggregation, proto.Trainable, dbg_name); + + var saved_device = proto.Device; + var load_with_device = _save_options.experimental_variable_policy.save_variable_devices() && !string.IsNullOrEmpty(saved_device); + + if (load_with_device) + { + tf.device(saved_device); + return (new UninitializedVariable( + shape: new Shape(proto.Shape.Dim.Select(x => (int)x.Size).ToArray()), + dtype: (TF_DataType)proto.Dtype, + name: name, + trainable: trainable, + aggregation: aggregation + ), setattr); + } + else + { + return (new UninitializedVariable( + shape: new Shape(proto.Shape.Dim.Select(x => (int)x.Size).ToArray()), + dtype: (TF_DataType)proto.Dtype, + name: name, + trainable: trainable, + aggregation: aggregation + ), setattr); + } + } + + private (ConcreteFunction, Action) _recreate_bare_concrete_function(SavedBareConcreteFunction proto, + Dictionary, Trackable> dependencies) + { + throw new NotImplementedException(); + //var fn = function_deserialization.setup_bare_concrete_function(proto, ) + } + + // TODO: remove this to a common class. + public static Action setattr = (x, y, z) => + { + Debug.Assert(y is string); + var properties = x.GetType().GetProperties(); + foreach(var p in properties) + { + if((string)y == p.Name) + { + p.SetValue(x, z); + return; + } + } + // TODO(Rinne): check if the property has been set successfully. + //throw new ValueError($"Cannot find the property {y} of {x}."); + }; + + public class _UserObject: AutoTrackable + { + + } + } +} diff --git a/src/TensorFlowNET.Core/Training/Saving/SavedModel/loader.static.cs b/src/TensorFlowNET.Core/Training/Saving/SavedModel/loader.static.cs new file mode 100644 index 000000000..a92cb5509 --- /dev/null +++ b/src/TensorFlowNET.Core/Training/Saving/SavedModel/loader.static.cs @@ -0,0 +1,122 @@ +using Google.Protobuf; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using Tensorflow.Checkpoint; +using Tensorflow.Operations; +using Tensorflow.Train; +using static Tensorflow.Binding; + +namespace Tensorflow +{ + public partial class Loader + { + public static SavedModel parse_saved_model(string export_dir) + { + var path_to_pbtxt = tf.io.gfile.join(export_dir, Constants.SAVED_MODEL_FILENAME_PBTXT); + var path_to_pb = tf.io.gfile.join(export_dir, Constants.SAVED_MODEL_FILENAME_PB); + + SavedModel saved_model = new SavedModel(); + if (File.Exists(path_to_pb)) + { + byte[] file_content; + using(var f = new FileStream(path_to_pb, FileMode.Open, FileAccess.Read)) + { + file_content = new byte[f.Length]; + Debug.Assert(f.Length <= int.MaxValue); + f.Read(file_content, 0, (int)f.Length); + } + // TODO: change to stream mode. + saved_model.MergeFrom(file_content); + return saved_model; + } + else if (File.Exists(path_to_pbtxt)) + { + throw new NotImplementedException(); + } + else + { + throw new IOException($"SavedModel file does not exist at: {export_dir}{Path.PathSeparator}" + + $"{{{Constants.SAVED_MODEL_FILENAME_PBTXT}|{Constants.SAVED_MODEL_FILENAME_PB}}}"); + } + } + + // TODO: revise the type of `tags` + public static Trackable load(string export_dir, object? tags = null, LoadOptions? options = null) + { + return load_partial(export_dir, null, tags, options)["root"]; + } + + public static IDictionary load_partial(string export_dir, IDictionary)>? filters, object? tags = null, LoadOptions? options = null) + { + if (options is null) + { + options = new LoadOptions(); + } + if (tags is not null) + { + throw new NotImplementedException(); + } + var (saved_model_proto, debug_info) = Loader.parse_saved_model_with_debug_info(export_dir); + + Trackable root = null; + Loader loader = null; + if (saved_model_proto.MetaGraphs.Count == 1 && saved_model_proto.MetaGraphs[0].ObjectGraphDef is not null) + { + // skip python code: `metrics.IncrementReadApi(_LOAD_V2_LABEL)` + var meta_graph_def = saved_model_proto.MetaGraphs[0]; + if (!BitConverter.IsLittleEndian) + { + SavedModelUtils.swap_function_tensor_content(meta_graph_def); + } + + var object_graph_proto = meta_graph_def.ObjectGraphDef; + var ckpt_options = new CheckpointOptions(options.experimental_io_device); + tf_with(ops.init_scope(), x => + { + loader = new Loader(object_graph_proto, saved_model_proto, export_dir, ckpt_options, options, filters); + root = loader.get(0); + // skip the assignment of `graph_debug_info`. + }); + // skip the assignment of `tensorflow_version` + // skip the assignment of `tensorflow_git_version` + // skip the process of `metrics`. + } + else + { + if(filters is not null && filters.Count > 0) + { + throw new ValueError("SavedModels saved from Tensorflow 1.x or Estimator (any" + + " version) cannot be loaded with node filters."); + } + tf_with(ops.init_scope(), x => + { + throw new NotImplementedException("Not implemented, please submit an issue to https://github.com/SciSharp/TensorFlow.NET/issues."); + }); + } + if(filters != null && filters.Count > 0) + { + return filters.Keys.ToDictionary(x => x, x => loader.get(x)); + } + else + { + var res = new Dictionary(); + res["root"] = root; + return res; + } + } + + public static (SavedModel, object?) parse_saved_model_with_debug_info(string export_dir) + { + var saved_model = parse_saved_model(export_dir); + + // TODO: implement debug info. + + return (saved_model, null); + } + + } +} diff --git a/src/TensorFlowNET.Core/Training/Saving/SavedModel/save.cs b/src/TensorFlowNET.Core/Training/Saving/SavedModel/save.cs index 94760e3df..4313920f5 100644 --- a/src/TensorFlowNET.Core/Training/Saving/SavedModel/save.cs +++ b/src/TensorFlowNET.Core/Training/Saving/SavedModel/save.cs @@ -6,7 +6,6 @@ using Google.Protobuf; using Tensorflow.Checkpoint; using Tensorflow.Functions; -using Tensorflow.ModelSaving; using Tensorflow.Train; using Tensorflow.Exceptions; using static Tensorflow.Binding; diff --git a/src/TensorFlowNET.Core/Training/Saving/SavedModel/save_context.cs b/src/TensorFlowNET.Core/Training/Saving/SavedModel/save_context.cs index 4cfe0b69b..47d8cbab9 100644 --- a/src/TensorFlowNET.Core/Training/Saving/SavedModel/save_context.cs +++ b/src/TensorFlowNET.Core/Training/Saving/SavedModel/save_context.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Text; -using Tensorflow.ModelSaving; namespace Tensorflow.Training.Saving.SavedModel { diff --git a/src/TensorFlowNET.Core/Training/Saving/saveable_object_util.py.cs b/src/TensorFlowNET.Core/Training/Saving/saveable_object_util.py.cs index a6e21e3e5..208311229 100644 --- a/src/TensorFlowNET.Core/Training/Saving/saveable_object_util.py.cs +++ b/src/TensorFlowNET.Core/Training/Saving/saveable_object_util.py.cs @@ -68,6 +68,34 @@ public static MySaveableObject[] validate_and_slice_inputs(IVariableV1[] names_t return saveables.ToArray(); } + public static MySaveableObject[] validate_and_slice_inputs(Dictionary names_to_saveables) + { + var saveables = new List(); + var seen_ops = new List(); + + foreach (var (name, op) in enumerate(names_to_saveables)) + { + foreach (var converted_saveable_object in saveable_objects_for_op(op, name)) + _add_saveable(saveables, seen_ops, converted_saveable_object); + } + return saveables.ToArray(); + } + + public static MySaveableObject[] validate_and_slice_inputs(Dictionary names_to_saveables) + { + var saveables = new List(); + var seen_ops = new List(); + + foreach(var item in names_to_saveables.OrderBy(x => x.Key)) + { + foreach(var converted_saveable_object in saveable_objects_for_op(item.Value, item.Key)) + { + _add_saveable(saveables, seen_ops, converted_saveable_object); + } + } + return saveables.ToArray(); + } + private static void _add_saveable(List saveables, List seen_ops, T saveable) where T : MySaveableObject { if (seen_ops.Contains(saveable.op)) @@ -77,6 +105,15 @@ private static void _add_saveable(List saveables, List seen_ops, T seen_ops.Add(saveable.op); } + private static void _add_saveable(List saveables, List seen_ops, MySaveableObject saveable) + { + if (seen_ops.Contains(saveable.variable)) + throw new ValueError($"The same saveable will be restored with two names: {saveable.op.OriginalVar.Name}"); + + saveables.Add(saveable); + seen_ops.Add(saveable.variable); + } + /// /// Create `SaveableObject`s from an operation. Note that the `op` should not be implicitly converted from `Variable`. /// @@ -136,19 +173,20 @@ public static IEnumerable saveable_objects_for_op(Trackable ob { full_name = name + "_" + attr; } - if(factory.TryGet(out var variable)) + var op = factory(full_name); + if(op.TryGet(out var variable)) { - foreach (var op in saveable_objects_for_op(variable as Trackable, variable.Name)) + foreach (var v in saveable_objects_for_op(variable as Trackable, variable.Name)) { - yield return op; + yield return v; } } else { - var saveable = factory.GetValue(); - foreach (var op in saveable_objects_for_op(saveable, saveable.name)) + var saveable = op.GetValue(); + foreach (var v in saveable_objects_for_op(saveable, saveable.name)) { - yield return op; + yield return v; } } } @@ -214,20 +252,19 @@ public static Dictionary op_list_to_dict(IVariableV1[] op_list, return names_to_saveables; } - public static IDictionary> saveable_objects_from_trackable(Trackable obj) + public static IDictionary>> saveable_objects_from_trackable(Trackable obj) { // skip the process of type `PythonState` - if (trackable_has_serialize_to_tensor(obj)) + Maybe create_saveable(string name = "") { - var name = TrackableUtils.SERIALIZE_TO_TENSORS_NAME; // skip the case that `obj._serialize_to_tensors` is `ConcreteFunction`. var tensor_dict = obj.serialize_to_tensors(); List specs = new(); List local_names = new(); string prefix = SaveableCompat.get_saveable_name(obj) ?? ""; - foreach(var pair in tensor_dict) + foreach (var pair in tensor_dict) { var tensor_name = pair.Key; var maybe_tensor = pair.Value; @@ -235,9 +272,9 @@ public static IDictionary> string spec_name = name + TrackableUtils.escape_local_name(tensor_name); IDictionary internal_dict; - if(maybe_tensor.TryGet(out var tensor)) + if (maybe_tensor.TryGet(out var tensor)) { - internal_dict= new Dictionary(); + internal_dict = new Dictionary(); internal_dict[""] = tensor; } else @@ -245,13 +282,18 @@ public static IDictionary> internal_dict = maybe_tensor.GetValue>(); } - foreach(var item in internal_dict) + foreach (var item in internal_dict) { specs.Add(new SaveSpec(item.Value, item.Key, spec_name)); } } - Dictionary> res = new(); - res[name] = new TrackableSaveable(obj, specs, name, local_names, prefix); + return new TrackableSaveable(obj, specs, name, local_names, prefix); + } + + if (trackable_has_serialize_to_tensor(obj)) + { + Dictionary>> res = new(); + res[TrackableUtils.SERIALIZE_TO_TENSORS_NAME] = create_saveable; return res; } else @@ -333,6 +375,28 @@ public static Func return restored_ops; }; } + + /// + /// Returns a dict of SaveableObject factories generated from loaded fns. + /// + /// + /// + public static IDictionary>> recreate_saveable_objects( + IDictionary saveable_fn_by_name, IEnumerable? temp_session) + { + if (saveable_fn_by_name.Count > 0) + { + throw new NotImplementedException("Not implemented, please submit an issue to https://github.com/SciSharp/TensorFlow.NET/issues"); + } + var res = new Dictionary>>(); + return res; + } + + public static Maybe create_saveable_object(string name, string key, Func> factory, + bool call_with_mapped_captures = false) + { + return factory(key); + } } public class SaveableCompatibilityConverter: Trackable diff --git a/src/TensorFlowNET.Core/Training/Trackable.cs b/src/TensorFlowNET.Core/Training/Trackable.cs index 132571f2a..7c86a5802 100644 --- a/src/TensorFlowNET.Core/Training/Trackable.cs +++ b/src/TensorFlowNET.Core/Training/Trackable.cs @@ -20,8 +20,8 @@ limitations under the License. using System.Linq; using Tensorflow.Checkpoint; using Tensorflow.Keras.Saving.SavedModel; -using Tensorflow.ModelSaving; using Tensorflow.Training; +using Tensorflow.Training.Saving.SavedModel; using static Tensorflow.Binding; namespace Tensorflow.Train @@ -41,9 +41,10 @@ public static class Constants protected IDictionary _unconditional_dependency_names; protected IList _unconditional_checkpoint_dependencies; + protected Dictionary> _unconditional_deferred_dependencies; - protected IDictionary> _self_saveable_object_factories = - new Dictionary>(); + protected IDictionary>> _self_saveable_object_factories = + new Dictionary>>(); private bool _manual_tracking = true; private static Trackable _none = new AutoTrackable(); @@ -71,6 +72,18 @@ public virtual string ObjectIdentifier public IList UnconditionalCheckpointDependencies { get => _unconditional_checkpoint_dependencies; } public IDictionary UnconditionalDependencyNames { get => _unconditional_dependency_names; } public IList CheckpointDependencies { get => UnconditionalCheckpointDependencies; } + public Dictionary> DeferredDependencies => _unconditional_deferred_dependencies; + public IDictionary>> SelfSaveableObjectFactories + { + get + { + return _self_saveable_object_factories; + } + set + { + _self_saveable_object_factories = value; + } + } /// /// Restore-on-create for a variable be saved with this `Checkpointable`. @@ -136,9 +149,11 @@ public void _maybe_initialize_trackable() _self_update_uid = -1; _unconditional_checkpoint_dependencies = new List(); _unconditional_dependency_names = new Dictionary(); + _unconditional_deferred_dependencies = new Dictionary>(); } - public virtual IDictionary _trackable_children(SaveType save_type, IDictionary>? cache) + public virtual IDictionary _trackable_children(SaveType save_type = SaveType.CHECKPOINT, + IDictionary>? cache = null) { _maybe_initialize_trackable(); return _unconditional_checkpoint_dependencies.ToDictionary(x => x.Name, x => x.Refer); @@ -174,10 +189,19 @@ public virtual Trackable _track_trackable(Trackable trackable, string name, bool /// public virtual void _handle_deferred_dependencies(string name, Trackable trackable) { - //_maybe_initialize_trackable(); - //trackable._maybe_initialize_trackable(); - - // TODO: complete the implementation. + _maybe_initialize_trackable(); + trackable._maybe_initialize_trackable(); + + if(_unconditional_deferred_dependencies.TryGetValue(name, out var dependencies)) + { + _unconditional_deferred_dependencies.Remove(name); + foreach(var checkpoint_position in dependencies.OrderByDescending(x => x.Checkpoint.RestoreUid)) + { + checkpoint_position.restore(trackable); + } + } + + // TODO(Rinne): deal with `_self_name_based_restores` } public virtual Trackable? _lookup_dependency(string name) @@ -225,12 +249,19 @@ public virtual List export_to_saved_model_graph(IDictionary> gather_saveables_for_checkpoint() + public virtual IDictionary>> gather_saveables_for_checkpoint() { + Maybe create_saveable(string name = "") + { + throw new NotImplementedException(); + //return new TrackableSaveable(this, null, name, null, null); + } if (saveable_object_util.trackable_has_serialize_to_tensor(this)) { // TODO: complete the implementation (need to complete the class `saveable_object_util.TrackableSaveable`). - throw new NotImplementedException(); + Dictionary>> res = new(); + res[""] = create_saveable; + return res; } else { @@ -259,4 +290,6 @@ public virtual IDictionary _restore_from_tensors(IDictionary< } public record class TrackableReference(string Name, Trackable Refer); + + public record class SlotVariableRestoration(int OptimizerId, int SlotVariableId, string SlotName); } diff --git a/src/TensorFlowNET.Core/Training/TrackableUtils.cs b/src/TensorFlowNET.Core/Training/TrackableUtils.cs index 390d95c75..05c513a83 100644 --- a/src/TensorFlowNET.Core/Training/TrackableUtils.cs +++ b/src/TensorFlowNET.Core/Training/TrackableUtils.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using Tensorflow.Checkpoint; using Tensorflow.Exceptions; using Tensorflow.Train; @@ -20,9 +21,9 @@ public CyclicDependencyError(IDictionary> leftover_dependency_map LeftOverDependencyMap = leftover_dependency_map.ToDictionary(x => x.Key, x => x.Value.AsEnumerable()); } } - private static string _ESCAPE_CHAR = "."; - private static string _OPTIMIZER_SLOTS_NAME = _ESCAPE_CHAR + "OPTIMIZER_SLOT"; - private static string OBJECT_ATTRIBUTES_NAME = _ESCAPE_CHAR + "ATTRIBUTES"; + internal static string _ESCAPE_CHAR = "."; + internal static string _OPTIMIZER_SLOTS_NAME = _ESCAPE_CHAR + "OPTIMIZER_SLOT"; + internal static string OBJECT_ATTRIBUTES_NAME = _ESCAPE_CHAR + "ATTRIBUTES"; internal static string SERIALIZE_TO_TENSORS_NAME = _ESCAPE_CHAR + "TENSORS"; public static string object_path_to_string(IEnumerable node_path_arr) { diff --git a/src/TensorFlowNET.Core/Variables/BaseResourceVariable.cs b/src/TensorFlowNET.Core/Variables/BaseResourceVariable.cs index 4005d5640..9b8cfcb5f 100644 --- a/src/TensorFlowNET.Core/Variables/BaseResourceVariable.cs +++ b/src/TensorFlowNET.Core/Variables/BaseResourceVariable.cs @@ -5,9 +5,9 @@ using Tensorflow.Train; using static Tensorflow.Binding; using System.Collections.Generic; -using Tensorflow.ModelSaving; using System.Diagnostics; using Tensorflow.Checkpoint; +using Tensorflow.Training.Saving.SavedModel; namespace Tensorflow { @@ -19,7 +19,11 @@ public class BaseResourceVariable : DisposableTrackableObject protected TF_DataType _dtype; public TF_DataType dtype => _dtype; protected string _handle_name; - protected string handle_name => _handle_name; + public string handle_name + { + get { return _handle_name; } + set { _handle_name = value; } + } protected string _unique_id; public string UniqueId => _unique_id; @@ -289,10 +293,10 @@ public virtual void write_object_proto(SavedObject proto, SaveOptions options) resource_variable_ops.write_object_proto_for_resource_variable(this, proto, options); } - public override IDictionary> gather_saveables_for_checkpoint() + public override IDictionary>> gather_saveables_for_checkpoint() { - var res = new Dictionary>(); - res[Trackable.Constants.VARIABLE_VALUE_KEY] = this; + var res = new Dictionary>>(); + res[Trackable.Constants.VARIABLE_VALUE_KEY] = x => this; return res; } diff --git a/src/TensorFlowNET.Core/Variables/ResourceVariable.cs b/src/TensorFlowNET.Core/Variables/ResourceVariable.cs index 1645d7130..3b1f1e968 100644 --- a/src/TensorFlowNET.Core/Variables/ResourceVariable.cs +++ b/src/TensorFlowNET.Core/Variables/ResourceVariable.cs @@ -238,5 +238,23 @@ public NDArray eval(Session session = null) { return _graph_element.eval(session); } + + public static (VariableSynchronization, VariableAggregation, bool) validate_synchronization_aggregation_trainable( + VariableSynchronization? synchronization, VariableAggregation? aggregation, bool? trainable, string name) + { + if(aggregation is null) + { + aggregation = VariableAggregation.None; + } + if(synchronization is null) + { + synchronization = VariableSynchronization.Auto; + } + if (trainable is null) + { + trainable = synchronization != VariableSynchronization.OnRead; + } + return (synchronization.Value, aggregation.Value, trainable.Value); + } } } diff --git a/src/TensorFlowNET.Keras/Engine/Functional.FromConfig.cs b/src/TensorFlowNET.Keras/Engine/Functional.FromConfig.cs index b0d1b2b6b..f4407265c 100644 --- a/src/TensorFlowNET.Keras/Engine/Functional.FromConfig.cs +++ b/src/TensorFlowNET.Keras/Engine/Functional.FromConfig.cs @@ -24,10 +24,10 @@ public static Functional from_config(ModelConfig config) /// /// /// - static (Tensors, Tensors, Dictionary) reconstruct_from_config(ModelConfig config) + public static (Tensors, Tensors, Dictionary) reconstruct_from_config(ModelConfig config, Dictionary? created_layers = null) { // Layer instances created during the graph reconstruction process. - var created_layers = new Dictionary(); + created_layers = created_layers ?? new Dictionary(); var node_index_map = new Dictionary<(string, int), int>(); var node_count_by_layer = new Dictionary(); var unprocessed_nodes = new Dictionary(); @@ -88,12 +88,7 @@ static void process_layer(Dictionary created_layers, layer = created_layers[layer_name]; else { - layer = layer_data.ClassName switch - { - "InputLayer" => InputLayer.from_config(layer_data.Config), - "Dense" => Dense.from_config(layer_data.Config), - _ => throw new NotImplementedException("") - }; + layer = generic_utils.deserialize_keras_object(layer_data.ClassName, layer_data.Config); created_layers[layer_name] = layer; } diff --git a/src/TensorFlowNET.Keras/Engine/Functional.cs b/src/TensorFlowNET.Keras/Engine/Functional.cs index 44eaef534..33320101b 100644 --- a/src/TensorFlowNET.Keras/Engine/Functional.cs +++ b/src/TensorFlowNET.Keras/Engine/Functional.cs @@ -53,6 +53,11 @@ public Functional(Tensors inputs, Tensors outputs, string name = null) Inputs = inputs, Outputs = outputs }) + { + Initialize(inputs, outputs, name); + } + + internal void Initialize(Tensors inputs, Tensors outputs, string name = null) { _input_layers = new List(); _output_layers = new List(); @@ -70,7 +75,14 @@ protected void _init_graph_network(Tensors inputs, Tensors outputs) this.inputs = inputs; this.outputs = outputs; built = true; - _buildInputShape = inputs.shape; + if(inputs.Length > 0) + { + _buildInputShape = inputs.shape; + } + else + { + _buildInputShape = new Saving.TensorShapeConfig(); + } if (outputs.Any(x => x.KerasHistory == null)) base_layer_utils.create_keras_history(outputs); diff --git a/src/TensorFlowNET.Keras/Engine/Layer.Layers.cs b/src/TensorFlowNET.Keras/Engine/Layer.Layers.cs index a2d212cb3..81fc26355 100644 --- a/src/TensorFlowNET.Keras/Engine/Layer.Layers.cs +++ b/src/TensorFlowNET.Keras/Engine/Layer.Layers.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; namespace Tensorflow.Keras.Engine { @@ -14,5 +15,30 @@ protected void StackLayers(params ILayer[] layers) public virtual Shape ComputeOutputShape(Shape input_shape) => throw new NotImplementedException(""); + + protected List _gather_children_variables(bool include_trainable = false, bool include_non_trainable = false) + { + List res = new(); + var nested_layers = _flatten_layers(false, false); + foreach (var layer in nested_layers) + { + if (layer is Layer l) + { + if (include_trainable == true && include_non_trainable == true) + { + res.AddRange(l.Variables); + } + else if (include_trainable == true && include_non_trainable == false) + { + res.AddRange(l.TrainableVariables); + } + else if(include_trainable == false && include_non_trainable == true) + { + res.AddRange(l.NonTrainableVariables); + } + } + } + return res; + } } } diff --git a/src/TensorFlowNET.Keras/Engine/Layer.Serialize.cs b/src/TensorFlowNET.Keras/Engine/Layer.Serialize.cs index fc405d872..ed5c2de0a 100644 --- a/src/TensorFlowNET.Keras/Engine/Layer.Serialize.cs +++ b/src/TensorFlowNET.Keras/Engine/Layer.Serialize.cs @@ -12,7 +12,7 @@ public abstract partial class Layer public override string ObjectIdentifier => TrackableSavedModelSaver.ObjectIdentifier; - public string TrackingMetadata => TrackableSavedModelSaver.TrackingMetadata; + public string GetTrackingMetadata() => TrackableSavedModelSaver.TrackingMetadata; public override IDictionary _trackable_children(SaveType save_type = SaveType.CHECKPOINT, IDictionary>? cache = null) { diff --git a/src/TensorFlowNET.Keras/Engine/Layer.cs b/src/TensorFlowNET.Keras/Engine/Layer.cs index 31b37d681..3934950bd 100644 --- a/src/TensorFlowNET.Keras/Engine/Layer.cs +++ b/src/TensorFlowNET.Keras/Engine/Layer.cs @@ -14,6 +14,7 @@ You may obtain a copy of the License at limitations under the License. ******************************************************************************/ +using Newtonsoft.Json.Linq; using System; using System.Collections.Generic; using System.Linq; @@ -66,16 +67,74 @@ public abstract partial class Layer : AutoTrackable, ILayer public bool SupportsMasking { get; set; } protected List _trainable_weights; - public virtual List TrainableVariables => _trainable_weights; + public virtual List TrainableVariables => TrainableWeights; protected List _non_trainable_weights; - public List non_trainable_variables => _non_trainable_weights; + public List NonTrainableVariables => NonTrainableWeights; + public List Variables => Weights; + + public virtual List TrainableWeights + { + get + { + if (!this.Trainable) + { + return new List(); + } + var children_weights = _gather_children_variables(true); + return children_weights.Concat(_trainable_weights).Distinct().ToList(); + } + } + + public virtual List NonTrainableWeights + { + get + { + if (!this.Trainable) + { + var children_weights = _gather_children_variables(true, true); + return children_weights.Concat(_trainable_weights).Concat(_non_trainable_weights).Distinct().ToList(); + } + else + { + var children_weights = _gather_children_variables(include_non_trainable: true); + return children_weights.Concat(_non_trainable_weights).Distinct().ToList(); + } + } + } + + public virtual List Weights + { + get + { + return TrainableWeights.Concat(NonTrainableWeights).ToList(); + } + set + { + if (Weights.Count() != value.Count()) throw new ValueError( + $"You called `set_weights` on layer \"{this.name}\"" + + $"with a weight list of length {len(value)}, but the layer was " + + $"expecting {len(Weights)} weights."); + foreach (var (this_w, v_w) in zip(Weights, value)) + this_w.assign(v_w, read_value: true); + } + } protected int id; public int Id => id; protected string name; protected string base_name; - public string Name => name; + public string Name + { + get + { + return name; + } + set + { + name = value; + } + } protected bool computePreviousMask; protected List updates; @@ -85,10 +144,11 @@ public abstract partial class Layer : AutoTrackable, ILayer List inboundNodes; public List InboundNodes => inboundNodes; - List outboundNodes; public List OutboundNodes => outboundNodes; + public JObject SerializedAttributes { get; set; } + ThreadLocal callContext = new ThreadLocal(); public CallContext CallContext => callContext.Value; public Tensor[] input @@ -117,6 +177,11 @@ public Shape OutputShape protected List _self_tracked_trackables; public Layer(LayerArgs args) + { + Initialize(args); + } + + internal virtual void Initialize(LayerArgs args) { this.args = args; // A stateful layer is a layer whose updates are run during inference too, @@ -273,46 +338,9 @@ protected virtual void _init_set_name(string name, bool zero_based = true) public int count_params() { if (Trainable) - return layer_utils.count_params(this, weights); + return layer_utils.count_params(this, Weights); return 0; } - List ILayer.TrainableWeights - { - get - { - return _trainable_weights; - } - } - - List ILayer.NonTrainableWeights - { - get - { - return _non_trainable_weights; - } - } - - public List weights - { - get - { - var weights = new List(); - weights.AddRange(_trainable_weights); - weights.AddRange(_non_trainable_weights); - return weights; - } - set - { - if (weights.Count() != value.Count()) throw new ValueError( - $"You called `set_weights` on layer \"{this.name}\"" + - $"with a weight list of length {len(value)}, but the layer was " + - $"expecting {len(weights)} weights."); - foreach (var (this_w, v_w) in zip(weights, value)) - this_w.assign(v_w, read_value: true); - } - } - - public List Variables => weights; public virtual IKerasConfig get_config() => args; diff --git a/src/TensorFlowNET.Keras/Engine/Model.Save.cs b/src/TensorFlowNET.Keras/Engine/Model.Save.cs index a1e891f98..a3956cccc 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.Save.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.Save.cs @@ -33,7 +33,7 @@ public void save(string filepath, { using (SharedObjectSavingScope.Enter()) { - KerasSavedModelUtils.Save(this, filepath, overwrite, include_optimizer, signatures, options, save_traces); + KerasSavedModelUtils.save_model(this, filepath, overwrite, include_optimizer, signatures, options, save_traces); } } } diff --git a/src/TensorFlowNET.Keras/Engine/Model.cs b/src/TensorFlowNET.Keras/Engine/Model.cs index dd3e11a27..bbc6e8293 100644 --- a/src/TensorFlowNET.Keras/Engine/Model.cs +++ b/src/TensorFlowNET.Keras/Engine/Model.cs @@ -36,6 +36,8 @@ public partial class Model : Layer, IModel IVariableV1 _predict_counter; bool _base_model_initialized; bool stop_training; + + public bool IsGraphNetwork => _is_graph_network; public OptimizerV2 Optimizer { @@ -49,6 +51,12 @@ public Model(ModelArgs args) _init_batch_counters(); } + internal override void Initialize(LayerArgs args) + { + _init_batch_counters(); + base.Initialize(args); + } + void _configure_steps_per_execution(int steps_per_execution) { _steps_per_execution = tf.Variable(steps_per_execution, @@ -81,10 +89,11 @@ void _init_batch_counters() public override List Layers => _flatten_layers(recursive: false, include_self: false).ToList(); - public override List TrainableVariables + public override List TrainableWeights { get { + // skip the assertion of weights created. var variables = new List(); if (!Trainable) @@ -95,18 +104,40 @@ public override List TrainableVariables foreach (var trackable_obj in _self_tracked_trackables) { if (trackable_obj.Trainable) - variables.AddRange(trackable_obj.TrainableVariables); + variables.AddRange(trackable_obj.TrainableWeights); } - foreach (var layer in _self_tracked_trackables) + variables.AddRange(_trainable_weights); + + return variables.Distinct().ToList(); + } + } + + public override List NonTrainableWeights + { + get + { + // skip the assertion of weights created. + var variables = new List(); + + foreach (var trackable_obj in _self_tracked_trackables) { - if (layer.Trainable) - variables.AddRange(layer.TrainableVariables); + variables.AddRange(trackable_obj.NonTrainableWeights); } - // variables.AddRange(_trainable_weights); + if (!Trainable) + { + var trainable_variables = new List(); + foreach (var trackable_obj in _self_tracked_trackables) + { + variables.AddRange(trackable_obj.TrainableWeights); + } + variables.AddRange(trainable_variables); + variables.AddRange(_trainable_weights); + variables.AddRange(_non_trainable_weights); + } - return variables; + return variables.Distinct().ToList(); } } diff --git a/src/TensorFlowNET.Keras/Engine/Sequential.cs b/src/TensorFlowNET.Keras/Engine/Sequential.cs index 4d87659bd..69665388b 100644 --- a/src/TensorFlowNET.Keras/Engine/Sequential.cs +++ b/src/TensorFlowNET.Keras/Engine/Sequential.cs @@ -44,8 +44,6 @@ public Sequential(SequentialArgs args) : base(args.Inputs, args.Outputs, name: args.Name) { this.args = args; - if (args.Layers == null) - args.Layers = new List(); // SupportsMasking = true; _compute_output_and_mask_jointly = true; _auto_track_sub_layers = false; @@ -54,10 +52,17 @@ public Sequential(SequentialArgs args) _created_nodes = new List(); // Add to the model any layers passed to the constructor. - if (args.Layers != null) + if (args.Layers is not null) { - foreach (var layer in args.Layers) - add(layer); + InitLayers(args.Layers); + } + } + + public void InitLayers(IEnumerable layers) + { + foreach(var layer in layers) + { + add(layer); } } diff --git a/src/TensorFlowNET.Keras/Layers/Activation/ELU.cs b/src/TensorFlowNET.Keras/Layers/Activation/ELU.cs index 45f64720f..9cb5b7565 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/ELU.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/ELU.cs @@ -25,8 +25,7 @@ public override void build(Shape input_shape) { throw new ValueError("Alpha must be a number greater than 0."); } - _buildInputShape = input_shape; - built = true; + base.build(input_shape); } protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) diff --git a/src/TensorFlowNET.Keras/Layers/Activation/Exponential.cs b/src/TensorFlowNET.Keras/Layers/Activation/Exponential.cs index 2fd2caee1..981f96f0b 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/Exponential.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/Exponential.cs @@ -14,8 +14,7 @@ public Exponential(LayerArgs args) : base(args) } public override void build(Shape input_shape) { - _buildInputShape = input_shape; - built = true; + base.build(input_shape); } protected override Tensors Call(Tensors inputs, Tensor state = null, bool? training = null) { diff --git a/src/TensorFlowNET.Keras/Layers/Activation/SELU.cs b/src/TensorFlowNET.Keras/Layers/Activation/SELU.cs index 1ef8d0e58..9b5bc0e66 100644 --- a/src/TensorFlowNET.Keras/Layers/Activation/SELU.cs +++ b/src/TensorFlowNET.Keras/Layers/Activation/SELU.cs @@ -19,8 +19,7 @@ public override void build(Shape input_shape) { if ( alpha < 0f ) { throw new ValueError("Alpha must be a number greater than 0."); } - _buildInputShape = input_shape; - built = true; + base.build(input_shape); } protected override Tensors Call ( Tensors inputs, Tensor state = null, bool? training = null ) { Tensor output = inputs; diff --git a/src/TensorFlowNET.Keras/Layers/Core/Dense.cs b/src/TensorFlowNET.Keras/Layers/Core/Dense.cs index ca8007d09..56fde9f2c 100644 --- a/src/TensorFlowNET.Keras/Layers/Core/Dense.cs +++ b/src/TensorFlowNET.Keras/Layers/Core/Dense.cs @@ -85,10 +85,5 @@ protected override Tensors Call(Tensors inputs, Tensor state = null, bool? train return outputs; } - - public static Dense from_config(LayerArgs args) - { - return new Dense(args as DenseArgs); - } } } diff --git a/src/TensorFlowNET.Keras/Layers/Core/InputLayer.cs b/src/TensorFlowNET.Keras/Layers/Core/InputLayer.cs index 03b4b742a..a44c0bded 100644 --- a/src/TensorFlowNET.Keras/Layers/Core/InputLayer.cs +++ b/src/TensorFlowNET.Keras/Layers/Core/InputLayer.cs @@ -102,11 +102,6 @@ public InputLayer(InputLayerArgs args) : name: Name); } - public static InputLayer from_config(LayerArgs args) - { - return new InputLayer(args as InputLayerArgs); - } - public override SavedModelSaver TrackableSavedModelSaver => new InputLayerSavedModelSaver(this); } } diff --git a/src/TensorFlowNET.Keras/Metrics/Metric.cs b/src/TensorFlowNET.Keras/Metrics/Metric.cs index 1dfc39c49..435eebd48 100644 --- a/src/TensorFlowNET.Keras/Metrics/Metric.cs +++ b/src/TensorFlowNET.Keras/Metrics/Metric.cs @@ -56,7 +56,7 @@ public virtual Tensor update_state(Tensor y_true, Tensor y_pred, Tensor sample_w public virtual void reset_states() { - foreach (var v in weights) + foreach (var v in Weights) v.assign(0); } diff --git a/src/TensorFlowNET.Keras/Models/ModelsApi.cs b/src/TensorFlowNET.Keras/Models/ModelsApi.cs index 73b77bc42..6597f5cdc 100644 --- a/src/TensorFlowNET.Keras/Models/ModelsApi.cs +++ b/src/TensorFlowNET.Keras/Models/ModelsApi.cs @@ -4,6 +4,7 @@ using System.Text; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Saving; +using Tensorflow.Keras.Saving.SavedModel; using ThirdParty.Tensorflow.Python.Keras.Protobuf; namespace Tensorflow.Keras.Models @@ -13,20 +14,9 @@ public class ModelsApi public Functional from_config(ModelConfig config) => Functional.from_config(config); - public void load_model(string filepath, bool compile = true) + public Model load_model(string filepath, bool compile = true, LoadOptions? options = null) { - var bytes = File.ReadAllBytes(Path.Combine(filepath, "saved_model.pb")); - var saved_mode = SavedModel.Parser.ParseFrom(bytes); - - var meta_graph_def = saved_mode.MetaGraphs[0]; - var object_graph_def = meta_graph_def.ObjectGraphDef; - - bytes = File.ReadAllBytes(Path.Combine(filepath, "keras_metadata.pb")); - var metadata = SavedMetadata.Parser.ParseFrom(bytes); - - // Recreate layers and metrics using the info stored in the metadata. - var keras_loader = new KerasObjectLoader(metadata, object_graph_def); - keras_loader.load_layers(compile: compile); + return KerasLoadModelUtils.load_model(filepath, compile: compile, options: options) as Model; } } } diff --git a/src/TensorFlowNET.Keras/Saving/KerasObjectLoader.cs b/src/TensorFlowNET.Keras/Saving/KerasObjectLoader.cs index fc8cab0c1..fffc2bac0 100644 --- a/src/TensorFlowNET.Keras/Saving/KerasObjectLoader.cs +++ b/src/TensorFlowNET.Keras/Saving/KerasObjectLoader.cs @@ -1,12 +1,24 @@ using Newtonsoft.Json; +using Newtonsoft.Json.Linq; using System; using System.Collections.Generic; +using System.ComponentModel; +using System.Diagnostics; using System.Linq; +using System.Reflection; using System.Text.RegularExpressions; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Layers; +using Tensorflow.Keras.Layers.Rnn; +using Tensorflow.Keras.Losses; +using Tensorflow.Keras.Metrics; +using Tensorflow.Keras.Saving.SavedModel; +using Tensorflow.Keras.Utils; +using Tensorflow.Train; +using Tensorflow.Training; using ThirdParty.Tensorflow.Python.Keras.Protobuf; +using static Tensorflow.ApiDef.Types; using static Tensorflow.Binding; using static Tensorflow.KerasApi; @@ -14,17 +26,29 @@ namespace Tensorflow.Keras.Saving { public class KerasObjectLoader { - SavedMetadata _metadata; - SavedObjectGraph _proto; - Dictionary _node_paths = new Dictionary(); - Dictionary model_layer_dependencies = new Dictionary(); - List _traversed_nodes_from_config = new List(); + private static readonly IDictionary PUBLIC_ATTRIBUTES = new CommonEndPoints().CheckpointableObjects; + private SavedMetadata _metadata; + private SavedObjectGraph _proto; + private Dictionary _node_paths = new Dictionary(); + private Dictionary model_layer_ids_dependencies = new Dictionary(); + private Dictionary model_layer_dependencies = new Dictionary(); + private List _traversed_nodes_from_config = new List(); + private Dictionary)> loaded_nodes; + private List _models_to_reconstruct; + public Dictionary)> LoadedNodes => loaded_nodes; + + static KerasObjectLoader() + { + PUBLIC_ATTRIBUTES[Keras.Saving.SavedModel.Constants.KERAS_ATTR] = null; + } public KerasObjectLoader(SavedMetadata metadata, SavedObjectGraph object_graph_def) { _metadata = metadata; _proto = object_graph_def; _metadata.Nodes.ToList().ForEach(x => _node_paths[x.NodeId] = x.NodePath); + _models_to_reconstruct = new List(); + loaded_nodes = new Dictionary)>(); } /// @@ -42,15 +66,255 @@ public void load_layers(bool compile = true) continue; } - _load_layer(node_metadata.NodeId, node_metadata.Identifier, node_metadata.Metadata); + loaded_nodes[node_metadata.NodeId] = _load_layer(node_metadata.NodeId, node_metadata.Identifier, node_metadata.Metadata); + } + foreach(var node_metadata in metric_list) + { + try + { + if (node_metadata.Identifier.Equals("_tf_keras_metric")) + { + continue; + } + loaded_nodes[node_metadata.NodeId] = _load_layer(node_metadata.NodeId, node_metadata.Identifier, + node_metadata.Metadata); + } + catch(ValueError e) + { + if (compile) + { + throw e; + } + // TODO: add logging.warning. + } + } + } + + public string get_path(int node_id) + { + return _node_paths[node_id]; + } + + /// + /// Finish setting up Keras objects. + /// + /// This function is executed after all objects and functions have been created. + /// Call functions and losses are attached to each layer, and once all layers + /// have been fully set up, graph networks are initialized. + /// + /// Subclassed models that are revived from the SavedModel are treated like + /// layers, and have their call/loss functions attached here. + /// + public void finalize_objects() + { + List layers_revived_from_config = new(); + List layers_revived_from_saved_model = new(); + foreach(var item in loaded_nodes) + { + var node_id = item.Key; + var node = item.Value.Item1; + if(node is not Layer || model_layer_ids_dependencies.ContainsKey(node_id)) + { + continue; + } + + _unblock_model_reconstruction(node_id, node as Layer); + + if(node is InputLayer or Metric) + { + continue; + } + + // TODO: deal with `RevivedLayer` and `RevivedInputLayer`. + layers_revived_from_config.Add(node as Layer); + } + + _finalize_saved_model_layers(layers_revived_from_saved_model); + _finalize_config_layers(layers_revived_from_config); + + _reconstruct_all_models(); + } + + private void _reconstruct_all_models() + { + HashSet all_initialized_models = new(); + for(int i = _models_to_reconstruct.Count - 1; i >= 0; i--) + { + int model_id = _models_to_reconstruct[i]; + all_initialized_models.Add(model_id); + var (model, layers) = model_layer_dependencies[model_id]; + _reconstruct_model(model_id, model, layers.ToList()); + _finalize_config_layers(new List() { model }); + } + + Debug.Assert(all_initialized_models.SequenceEqual(model_layer_dependencies.Keys)); + } + + private void _reconstruct_model(int model_id, Model model, List layers) + { + var config = JsonConvert.DeserializeObject(_metadata.Nodes[model_id].Metadata)["config"]; + + if(model.input is not null && model.input.Length > 0) + { + + } + else if(model is Sequential s) + { + if(layers is null || layers.Count == 0 || layers[0] is not InputLayer) + { + if (config["layers"][0]["class_name"].ToObject() == "InputLayer") + { + layers.Insert(0, new InputLayer(config["layers"][0]["config"].ToObject())); + } + else if (config["layers"][0]["config"]["batch_input_shape"] is not null) + { + // TODO(Rinne): implement it + } + } + + // `model.__init__(layers, config["name"])` + s.InitLayers(layers); + s.Name = config["name"].ToObject(); + if(s.input is null || s.input.Length == 0) + { + var first_layer = _get_child_layer_node_ids(model_id)[0]; + var input_specs = _infer_inputs(first_layer); + var input_shapes = _infer_inputs(first_layer, true); + // `model._set_inputs(input_specs)` + + // skip the check of input_specs is Dictionary + if (!s.Built) + { + s.build(input_shapes); + } + } + } + else + { + // skip the parameter `created_layers`. + var (inputs, outputs, created_layers) = Functional.reconstruct_from_config(generic_utils.deserialize_model_config(config), + layers.ToDictionary(x => x.Name, x => x as ILayer)); + // skip the `model.__init__` + (model as Functional).Initialize(inputs, outputs, config["name"].ToObject()); + (model as Functional).connect_ancillary_layers(created_layers); + } + + _set_network_attributes_from_metadata(model); + _unblock_model_reconstruction(model_id, model); + } + + private void _set_network_attributes_from_metadata(Model revived_object) + { + // TODO: implement it. + } + + /// + /// Runs the final steps of loading Keras Layers from config. + /// + /// + private void _finalize_config_layers(List layers) + { + foreach(var layer in layers) + { + if (_is_graph_network(layer)) + { + _restore_layer_unconditional_losses(layer); + } + _restore_layer_activation_loss(layer); + _restore_layer_metrics(layer); + + // TODO(Rinne): deal with RNN. + } + } + + /// + /// Runs the final steps of loading Keras Layers from SavedModel. + /// + /// + private void _finalize_saved_model_layers(List layers) + { + foreach(var layer in layers) + { + // TODO(Rinne): deal with `RevivedNetwork`. + + _restore_layer_unconditional_losses(layer); + _restore_layer_activation_loss(layer); + _restore_layer_metrics(layer); + } + } + + private void _restore_layer_unconditional_losses(Layer layer) + { + // TODO(Rinne): implement it. + } + + private void _restore_layer_activation_loss(Layer layer) + { + // TODO(Rinne): implement it. + } + + private void _restore_layer_metrics(Layer layer) + { + // TODO(Rinne): implement it. + } + + /// + /// Removes layer from blocking model reconstruction. + /// + /// + /// + private void _unblock_model_reconstruction(int layer_id, Layer layer) + { + foreach(var depencency in model_layer_ids_dependencies) + { + var layer_ids = depencency.Value.Item2; + var layers = model_layer_dependencies.SetDefault(depencency.Key, + (depencency.Value.Item1, new Layer[depencency.Value.Item2.Length])).Item2; + if (!layer_ids.Contains(layer_id)) + { + continue; + } + layers[Array.IndexOf(layer_ids, layer_id)] = layer; + if (layers.All(x => x is not null)) + { + _models_to_reconstruct.Add(depencency.Key); + } } } - void _load_layer(int node_id, string identifier, string metadata_json) + private (Trackable, Action) _load_layer(int node_id, string identifier, string metadata_json) { - metadata_json = metadata_json.Replace("\"dtype\": \"float32\"", "\"dtype\": 1"); var metadata = JsonConvert.DeserializeObject(metadata_json); - _revive_from_config(identifier, metadata, node_id); + + if (loaded_nodes.ContainsKey(node_id)) + { + var (node, setter) = loaded_nodes[node_id]; + + _maybe_add_serialized_attributes(node as Layer, metadata); + var config = metadata.Config; + if(_is_graph_network(node as Layer) && generic_utils.validate_config(config)) + { + Debug.Assert(node is Model); + var child_nodes = _get_child_layer_node_ids(node_id); + model_layer_ids_dependencies[node_id] = (node as Model, child_nodes); + if(child_nodes is null || child_nodes.Length == 0) + { + _models_to_reconstruct.Add(node_id); + } + } + return (node, setter); + } + else + { + var (obj, setter) = _revive_from_config(identifier, metadata, node_id); + if (obj is null) + { + (obj, setter) = _revive_custom_object(identifier, metadata); + } + Debug.Assert(obj is Layer); + _maybe_add_serialized_attributes(obj as Layer, metadata); + return (obj, setter); + } } /// @@ -59,11 +323,34 @@ void _load_layer(int node_id, string identifier, string metadata_json) /// /// /// - void _revive_from_config(string identifier, KerasMetaData metadata, int node_id) + private (Trackable, Action) _revive_from_config(string identifier, KerasMetaData metadata, int node_id) { - var obj = _revive_graph_network(identifier, metadata, node_id); - obj = obj ?? _revive_layer_or_model_from_config(metadata, node_id); + Trackable obj; + if(identifier == Keras.Saving.SavedModel.Constants.METRIC_IDENTIFIER) + { + // TODO(Rinne): implement it. + return (null, null); + //throw new NotImplementedException("Not implemented, please submit an issue to https://github.com/SciSharp/TensorFlow.NET/issues."); + } + else + { + obj = _revive_graph_network(identifier, metadata, node_id); + obj = obj ?? _revive_layer_or_model_from_config(metadata, node_id); + } + + if(obj is null) + { + return (null, null); + } + var setter = _config_node_setter(_revive_setter); _add_children_recreated_from_config(obj, _proto.Nodes[node_id], node_id); + return (obj, setter); + } + + private (Trackable, Action) _revive_custom_object(string identifier, KerasMetaData metadata) + { + // TODO(Rinne): implement it. + throw new NotImplementedException(); } Model _revive_graph_network(string identifier, KerasMetaData metadata, int node_id) @@ -71,6 +358,12 @@ Model _revive_graph_network(string identifier, KerasMetaData metadata, int node_ var config = metadata.Config; var class_name = metadata.ClassName; Model model = null; + + if(!metadata.IsGraphNetwork && class_name != "Sequential" && class_name != "Functional") + { + return null; + } + if (class_name == "Sequential") { model = new Sequential(new SequentialArgs @@ -78,34 +371,82 @@ Model _revive_graph_network(string identifier, KerasMetaData metadata, int node_ Name = config.GetValue("name").ToString() }); } - else if (class_name == "Functional") + else if(identifier == Keras.Saving.SavedModel.Constants.SEQUENTIAL_IDENTIFIER) { - throw new NotImplementedException(""); + model = new Sequential(new SequentialArgs + { + Name = class_name + }); + } + else + { + model = new Functional(new Tensors(), new Tensors(), config["name"].ToObject()); } - - if (!metadata.IsGraphNetwork) - return null; // Record this model and its layers. This will later be used to reconstruct // the model. var layers = _get_child_layer_node_ids(node_id); - model_layer_dependencies[node_id] = (model, layers); + model_layer_ids_dependencies[node_id] = (model, layers); + if(layers is null || layers.Length == 0) + { + _models_to_reconstruct.Add(node_id); + } return model; } - Model _revive_layer_or_model_from_config(KerasMetaData metadata, int node_id) + Layer _revive_layer_or_model_from_config(KerasMetaData metadata, int node_id) { var config = metadata.Config; var class_name = metadata.ClassName; var shared_object_id = metadata.SharedObjectId; var must_restore_from_config = metadata.MustRestoreFromConfig; - var obj = class_name switch - { - "Resizing" => Resizing.from_config(config), - _ => throw new NotImplementedException("") - }; + + var obj = generic_utils.deserialize_keras_object(class_name, config); + + obj.Name = metadata.Name; + // TODO(Rinne): add `trainable`, `dtype`, `stateful` and `save_spec` + + var built = _try_build_layer(obj, node_id, metadata.BuildInputShape); - return null; + if (!built) + { + return null; + } + return obj; + } + + private void _revive_setter(object layer, object name, object value) + { + Debug.Assert(name is string); + Debug.Assert(layer is Layer); + if(PUBLIC_ATTRIBUTES.ContainsKey(name as string)) + { + if(value is Trackable) + { + (layer as Layer)._track_trackable(value as Trackable, name as string); + } + if((layer as Layer).SerializedAttributes is null) + { + (layer as Layer).SerializedAttributes = new JObject(); + } + (layer as Layer).SerializedAttributes[name as string] = JToken.FromObject(value); + } + else if(layer is Functional && Regex.Match(name as string, @"^layer(_with_weights)?-[\d+]").Success) + { + (layer as Functional)._track_trackable(value as Trackable, name as string, overwrite: true); + } + else + { + var properties = layer.GetType().GetProperties(); + foreach(var p in properties) + { + if(p.Name == name as string && p.GetValue(layer) is not null) + { + return; + } + } + Loader.setattr(layer, name, value); + } } /// @@ -143,34 +484,186 @@ int[] _get_child_layer_node_ids(int node_id) /// /// /// - void _add_children_recreated_from_config(Model obj, SavedObject proto, int node_id) + void _add_children_recreated_from_config(Trackable obj, SavedObject proto, int node_id) { if (_traversed_nodes_from_config.Contains(node_id)) return; var parent_path = _node_paths[node_id]; _traversed_nodes_from_config.Add(node_id); - if (!obj.Built) + obj._maybe_initialize_trackable(); + + if(obj is Layer layer && !layer.Built) { - var metadata_json = proto.UserObject.Metadata.Replace("\"dtype\": \"float32\"", "\"dtype\": 1"); - var metadata = JsonConvert.DeserializeObject(metadata_json); - _try_build_layer(obj, node_id, metadata.BuildInputShape); + var metadata = JsonConvert.DeserializeObject(_metadata.Nodes[node_id].Metadata); + _try_build_layer(layer, node_id, metadata.BuildInputShape); + } + + + List<(Trackable, int, string)> children = new(); + foreach(var refer in proto.Children) + { + var obj_child = obj._lookup_dependency(refer.LocalName); + children.Add((obj_child, refer.NodeId, refer.LocalName)); + } + + var metric_list_node_id = _search_for_child_node(node_id, new string[] { + Keras.Saving.SavedModel.Constants.KERAS_ATTR, "layer_metrics" + }); + if(metric_list_node_id is not null && obj is Model model && model.metrics is not null) + { + var obj_metrics = model.metrics.ToDictionary(x => x.Name, x => x); + foreach(var refer in _proto.Nodes[metric_list_node_id.Value].Children) + { + if (obj_metrics.TryGetValue(refer.LocalName, out var metric)) + { + var metric_path = $"{Keras.Saving.SavedModel.Constants.KERAS_ATTR}.layer_metrics.{refer.LocalName}"; + children.Add((metric as Metric, refer.NodeId, metric_path)); + } + } + } + + foreach(var (obj_child, child_id, child_name) in children) + { + if(obj_child is null) + { + continue; + } + var child_proto = _proto.Nodes[child_id]; + + // skip the check for registered identifier + + Action setter; + if (Keras.Saving.SavedModel.Constants.KERAS_OBJECT_IDENTIFIERS.Contains(obj_child.ObjectIdentifier)) + { + setter = _revive_setter; + } + else + { + setter = Loader.setattr; + } + + if (loaded_nodes.ContainsKey(child_id)) + { + // skip the logging.warning + continue; + } + + if(child_proto.KindCase == SavedObject.KindOneofCase.Variable && !string.IsNullOrEmpty(child_proto.Variable.Name)) + { + (obj_child as BaseResourceVariable).handle_name = child_proto.Variable.Name + ":0"; + } + + if(obj_child is TrackableDataStructure) + { + setter = (x, y, z) => { }; + } + + var child_path = $"{parent_path}.{child_name}"; + _node_paths[child_id] = child_path; + _add_children_recreated_from_config(obj_child, child_proto, child_id); + loaded_nodes[child_id] = (obj_child, setter); } } - bool _try_build_layer(Model obj, int node_id, Shape build_input_shape) + private bool _try_build_layer(Layer obj, int node_id, Shape build_input_shape) { if (obj.Built) return true; + if(build_input_shape is null) + { + build_input_shape = _infer_inputs(node_id, convert_to_shapes: true); + } + + if(build_input_shape is not null) + { + obj.build(build_input_shape); + // In tf python here is a `base_layer.Layer.build(obj, build_input_shape)`. + // On the one hand, C# does not support call a method from specified parent class. + // On the other hand, currently All class derived from Layer call `Layer.Build` or + // move the implementation of `Layer.build` to its own `build` method. + // Therefore we do not call it here. + // However, it's still quite risky once in the future a certain class derived from + // `Layer` does not call `Layer.build`. + + return true; + } + return false; } - bool _try_build_layer(Layer obj, int node_id, Shape build_input_shape) + /// + /// Infers input shape of layer from SavedModel functions. + /// + /// + /// + /// + private Shape _infer_inputs(int layer_node_id, bool convert_to_shapes = false) { - if (obj.Built) - return true; + var call_fn_id = _search_for_child_node(layer_node_id, new string[] { "call_and_return_all_conditional_losses" }); + if(call_fn_id is null) + { + return null; + } + var concrete_functions = _proto.Nodes[call_fn_id.Value].Function.ConcreteFunctions; + if(concrete_functions is null) + { + return null; + } + var call_fn_name = concrete_functions[0]; + var call_fn_proto = _proto.ConcreteFunctions[call_fn_name]; + throw new NotImplementedException("Not implemented, please submit an issue to https://github.com/SciSharp/TensorFlow.NET/issues."); + } + + private int? _search_for_child_node(int parent_id, IEnumerable path_to_child) + { + if(path_to_child is null || path_to_child.Count() == 0) + { + return parent_id; + } + + foreach(var child in _proto.Nodes[parent_id].Children) + { + if(child.LocalName == path_to_child.First()) + { + return _search_for_child_node(child.NodeId, path_to_child.Skip(1)); + } + } + return null; + } + + private bool _is_graph_network(Layer layer) + { + // TODO: deal with `RevivedLayer` + if(layer is Functional) + { + return (layer as Functional).IsGraphNetwork || layer is Sequential; + } return false; } + + private void _maybe_add_serialized_attributes(Layer layer, KerasMetaData metadata) + { + // TODO: deal with `RevivedLayer` + } + + /// + /// Creates edges for nodes that are recreated from config. + /// + /// + private Action _config_node_setter(Action setter) + { + void setattr_wrapper(object obj, object name, object value) + { + Debug.Assert(obj is Trackable); + Debug.Assert(name is string); + if((obj as Trackable)._lookup_dependency(name as string) is null) + { + setter(obj, name, value); + } + } + return setattr_wrapper; + } } } diff --git a/src/TensorFlowNET.Keras/Saving/SavedModel/Save.cs b/src/TensorFlowNET.Keras/Saving/SavedModel/Save.cs index c7b7e52f4..220eae4b4 100644 --- a/src/TensorFlowNET.Keras/Saving/SavedModel/Save.cs +++ b/src/TensorFlowNET.Keras/Saving/SavedModel/Save.cs @@ -17,7 +17,7 @@ namespace Tensorflow.Keras.Saving.SavedModel; public partial class KerasSavedModelUtils { - public static void Save(Model model, string filepath, bool overwrite, bool include_optimizer, ConcreteFunction? signatures, + public static void save_model(Model model, string filepath, bool overwrite, bool include_optimizer, ConcreteFunction? signatures, SaveOptions? options, bool save_traces = true) { if (!overwrite && File.Exists(filepath)) @@ -95,7 +95,7 @@ public static SavedMetadata generate_keras_metadata(IList saved_nodes BadConsumers = { } }, Identifier = layer.ObjectIdentifier, - Metadata = layer.TrackingMetadata + Metadata = layer.GetTrackingMetadata() }; metadata.Nodes.Add(saved_object); @@ -130,7 +130,7 @@ public static IDictionary wrap_layer_objects(Layer layer, IDi if (x is ResourceVariable or RefVariable) return (Trackable)x; else throw new TypeError($"The type{x.GetType()} is not supported for the wrapping of layer."); })); - var non_trainable_variables = TrackableDataStructure.wrap_or_unwrap(layer.non_trainable_variables.Select(x => + var non_trainable_variables = TrackableDataStructure.wrap_or_unwrap(layer.NonTrainableVariables.Select(x => { if (x is ResourceVariable or RefVariable) return (Trackable)x; else throw new TypeError($"The type{x.GetType()} is not supported for the wrapping of layer."); diff --git a/src/TensorFlowNET.Keras/Saving/SavedModel/load.cs b/src/TensorFlowNET.Keras/Saving/SavedModel/load.cs new file mode 100644 index 000000000..abb2012f8 --- /dev/null +++ b/src/TensorFlowNET.Keras/Saving/SavedModel/load.cs @@ -0,0 +1,96 @@ +using Google.Protobuf; +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using Tensorflow.Keras.Engine; +using Tensorflow.Train; +using ThirdParty.Tensorflow.Python.Keras.Protobuf; +using static Tensorflow.Binding; +using static Tensorflow.KerasApi; + +namespace Tensorflow.Keras.Saving.SavedModel +{ + public class KerasLoadModelUtils + { + /// + /// Corresponding to keras/saving/save.py/load_model + /// + /// + /// + /// + /// + /// + public static Trackable load_model(string filepath, IDictionary? custom_objects = null, + bool compile = true, LoadOptions? options = null) + { + using (SharedObjectSavingScope.Enter()) + { + using (LoadContext.load_context(options)) + { + if (!File.Exists(filepath) && !Directory.Exists(filepath)) + { + throw new IOException($"No file or directory found at {filepath}."); + } + if (Directory.Exists(filepath)) + { + return load(filepath, compile, options); + } + else + { + throw new NotImplementedException("Model load of h5 format has not been supported. Please submit an issue to https://github.com/SciSharp/TensorFlow.NET/issues if it's needed."); + } + } + } + } + + private static Trackable load(string path, bool compile = true, LoadOptions? options = null) + { + SavedMetadata metadata = new SavedMetadata(); + var meta_graph_def = Loader.parse_saved_model(path).MetaGraphs[0]; + var object_graph_def = meta_graph_def.ObjectGraphDef; + string path_to_metadata_pb = Path.Combine(path, Constants.SAVED_METADATA_PATH); + if (File.Exists(path_to_metadata_pb)) + { + metadata.MergeFrom(new FileStream(path_to_metadata_pb, FileMode.Open, FileAccess.Read)); + } + else + { + throw new NotImplementedException("Not implemented, please submit an issue to https://github.com/SciSharp/TensorFlow.NET/issues."); + } + + if (metadata.Nodes is null || metadata.Nodes.Count == 0) + { + return Loader.load(path, options: options) as Model; + } + + var keras_loader = new KerasObjectLoader(metadata, object_graph_def); + keras_loader.load_layers(compile: compile); + + Dictionary)> nodes_to_load = new(); + nodes_to_load["root"] = (null, null); + foreach(var item in keras_loader.LoadedNodes) + { + nodes_to_load[keras_loader.get_path(item.Key)] = item.Value; + } + var loaded = Loader.load_partial(path, nodes_to_load, options); + + keras_loader.finalize_objects(); + // keras_loader.del_tracking(); + + var model = loaded["root"]; + + if(model is Model && compile) + { + // TODO(Rinne): implement it. + } + + if (!tf.Context.executing_eagerly()) + { + // TODO(Rinne): implement it. + } + + return model; + } + } +} diff --git a/src/TensorFlowNET.Keras/Saving/SavedModel/load_context.cs b/src/TensorFlowNET.Keras/Saving/SavedModel/load_context.cs new file mode 100644 index 000000000..11b1201d0 --- /dev/null +++ b/src/TensorFlowNET.Keras/Saving/SavedModel/load_context.cs @@ -0,0 +1,69 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using Tensorflow.Training.Saving.SavedModel; + +namespace Tensorflow.Keras.Saving.SavedModel +{ + // TODO: remove this class to common project. + public class ContextHandler: IDisposable + { + public Action DisposeCallBack { get; set; } + public void Dispose() + { + DisposeCallBack.Invoke(true); + } + } + public class LoadContext + { + private bool _entered_load_context; + private LoadOptions? _load_options; + private static ThreadLocal _load_context = new(); + private LoadContext() + { + _entered_load_context = false; + _load_options = null; + } + + public void set_load_options(LoadOptions load_options) + { + _load_options = load_options; + _entered_load_context = true; + } + + private void clear_load_options() + { + _load_options = null; + _entered_load_context = false; + } + + private LoadOptions? load_options() + { + return _load_options; + } + + public static ContextHandler load_context(LoadOptions? load_options) + { + if(_load_context.Value is null) + { + _load_context.Value = new LoadContext(); + } + _load_context.Value.set_load_options(load_options); + return new ContextHandler() + { + DisposeCallBack = _ => _load_context.Value.clear_load_options() + }; + } + + public static LoadOptions? get_load_option() + { + return _load_context.Value.load_options(); + } + + public static bool in_load_context() + { + return _load_context.Value._entered_load_context; + } + } +} diff --git a/src/TensorFlowNET.Keras/Utils/generic_utils.cs b/src/TensorFlowNET.Keras/Utils/generic_utils.cs index 730a33e3e..03acce0ca 100644 --- a/src/TensorFlowNET.Keras/Utils/generic_utils.cs +++ b/src/TensorFlowNET.Keras/Utils/generic_utils.cs @@ -19,15 +19,21 @@ limitations under the License. using System; using System.Collections; using System.Collections.Generic; +using System.Data; using System.Diagnostics; using System.Linq; +using System.Reflection; using Tensorflow.Keras.ArgsDefinition; +using Tensorflow.Keras.Engine; +using Tensorflow.Keras.Layers; using Tensorflow.Keras.Saving; +using Tensorflow.Train; namespace Tensorflow.Keras.Utils { public class generic_utils { + private static readonly string _LAYER_UNDEFINED_CONFIG_KEY = "layer was saved without config"; /// /// This method does not have corresponding method in python. It's close to `serialize_keras_object`. /// @@ -51,6 +57,58 @@ public static JObject serialize_keras_object(IKerasConfigable instance) return serialize_utils.serialize_keras_class_and_config(instance.GetType().Name, config, instance); } + public static Layer deserialize_keras_object(string class_name, JToken config) + { + var argType = Assembly.Load("Tensorflow.Binding").GetType($"Tensorflow.Keras.ArgsDefinition.{class_name}Args"); + var deserializationMethod = typeof(JToken).GetMethods(BindingFlags.Instance | BindingFlags.Public) + .Single(x => x.Name == "ToObject" && x.IsGenericMethodDefinition && x.GetParameters().Count() == 0); + var deserializationGenericMethod = deserializationMethod.MakeGenericMethod(argType); + var args = deserializationGenericMethod.Invoke(config, null); + var layer = Assembly.Load("Tensorflow.Keras").CreateInstance($"Tensorflow.Keras.Layers.{class_name}", true, BindingFlags.Default, null, new object[] { args }, null, null); + Debug.Assert(layer is Layer); + return layer as Layer; + } + + public static Layer deserialize_keras_object(string class_name, LayerArgs args) + { + var layer = Assembly.Load("Tensorflow.Keras").CreateInstance($"Tensorflow.Keras.Layers.{class_name}", true, BindingFlags.Default, null, new object[] { args }, null, null); + Debug.Assert(layer is Layer); + return layer as Layer; + } + + public static LayerArgs deserialize_layer_args(string class_name, JToken config) + { + var argType = Assembly.Load("Tensorflow.Binding").GetType($"Tensorflow.Keras.ArgsDefinition.{class_name}Args"); + var deserializationMethod = typeof(JToken).GetMethods(BindingFlags.Instance | BindingFlags.Public) + .Single(x => x.Name == "ToObject" && x.IsGenericMethodDefinition && x.GetParameters().Count() == 0); + var deserializationGenericMethod = deserializationMethod.MakeGenericMethod(argType); + var args = deserializationGenericMethod.Invoke(config, null); + Debug.Assert(args is LayerArgs); + return args as LayerArgs; + } + + public static ModelConfig deserialize_model_config(JToken json) + { + ModelConfig config = new ModelConfig(); + config.Name = json["name"].ToObject(); + config.Layers = new List(); + var layersToken = json["layers"]; + foreach (var token in layersToken) + { + var args = deserialize_layer_args(token["class_name"].ToObject(), token["config"]); + config.Layers.Add(new LayerConfig() + { + Config = args, + Name = token["name"].ToObject(), + ClassName = token["class_name"].ToObject(), + InboundNodes = token["inbound_nodes"].ToObject>() + }); + } + config.InputLayers = json["input_layers"].ToObject>(); + config.OutputLayers = json["output_layers"].ToObject>(); + return config; + } + public static string to_snake_case(string name) { return string.Concat(name.Select((x, i) => @@ -60,5 +118,15 @@ public static string to_snake_case(string name) x.ToString(); })).ToLower(); } + + /// + /// Determines whether config appears to be a valid layer config. + /// + /// + /// + public static bool validate_config(JObject config) + { + return !config.ContainsKey(_LAYER_UNDEFINED_CONFIG_KEY); + } } } diff --git a/src/TensorFlowNET.Keras/Utils/layer_utils.cs b/src/TensorFlowNET.Keras/Utils/layer_utils.cs index 3c38a6d1b..07d9f685e 100644 --- a/src/TensorFlowNET.Keras/Utils/layer_utils.cs +++ b/src/TensorFlowNET.Keras/Utils/layer_utils.cs @@ -104,7 +104,7 @@ public static void print_summary(Model model, int line_length = -1, float[] posi } var trainable_count = count_params(model, model.TrainableVariables); - var non_trainable_count = count_params(model, model.non_trainable_variables); + var non_trainable_count = count_params(model, model.NonTrainableVariables); print($"Total params: {trainable_count + non_trainable_count}"); print($"Trainable params: {trainable_count}"); diff --git a/test/TensorFlowNET.Keras.UnitTest/Assets/simple_model_from_auto_compile/bias0.npy b/test/TensorFlowNET.Keras.UnitTest/Assets/simple_model_from_auto_compile/bias0.npy new file mode 100644 index 0000000000000000000000000000000000000000..b5a8f8b32be6682b47718734f53ecfa2455db67d GIT binary patch literal 528 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1ZlV+l>qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= zXCxM+0{I$-1_nBsItsN4WCN~{PpfR3=1jM>_uFaH_-6l}!?k%=F%Q|f}vrpMRiFe%dSVPG+KY`!YcpbZqmdG9Jbqvh54PXWRZaa8K5!2YZAj-rQ3X(QCVNzx!Ut2Zpv^AL!VgR`=NZ zPUedBE-};H;+D(y8kJ43J+G9w=fuf>yWX6Z*sGU5eNR`AovkRZ?4C2+2lmW%j>eS7LjM{xI4u|DtUBQ|sjJcWa7mW07+O}+;W6K#k&nAX@$?p9-ENl#YH`|0XEVudSXm9(*u*J5l c`GW1PDOt9^FMHcA_0qTfZL@OskrPI?0B{}Ipa1{> literal 0 HcmV?d00001 diff --git a/test/TensorFlowNET.Keras.UnitTest/Assets/simple_model_from_auto_compile/fingerprint.pb b/test/TensorFlowNET.Keras.UnitTest/Assets/simple_model_from_auto_compile/fingerprint.pb new file mode 100644 index 0000000000000000000000000000000000000000..b62a57c3de0385cc28ae0a0a153308351a060ba8 GIT binary patch literal 55 zcmV-70LcFckM8Qir;LW1jR6pu_4@vg(9NKj0T}Y@wUOz^*yia0Adazt>V~k&2X+UAc>DYxztJ-=)Mu6t-RpVKXBMmKXlrZ8>1eBI$i>S2|4xwmK@V+D zn81nDN>(CH4Tlwrn11^~lo1HS7d_TYxoHsYDzw8Gt0nYfempczG}G*PXR&Ks7By3a zkz?JAnb#V?abE&nw-CYgatm-kG!BE(ZIF>1VK#Lt!N{}KuyWrkruL&3(N=K>oJ^p+ zCQqTqeIH%*bQ=jkHO{cXq#vX-<%#% z1wVbryRjZ}n`QCtvgef7r-puOJ%-OhS0h(K4t{q2LGH|30}>)?R6Tz+R0|wJE&e>5 zbL%%f+4Cp%w;2PgVFx2%h;>s%^r43uJk%+rQJbe2tq^@$$dd^7zRkz&Rd=ZHX$kh^ z-CPXXo)1CtsyJun3#YrM27^m^$R=kGvQEYWLt6i2Qob#Rj=MaVtndfT8(RZz%2(;z zi-s6edXOv^mZFQC)Jb81G)fgO$CmCsZj+5FP7m&aGtF-Z2!19OEefQS#}Ryur|DkF z0Wz^f9(*~~s9wB{?R%ev=fqtwz0917-+#fnR%GJsGIOF_stAiawn5OyW?Wz$59RAr zk-Iz#;`WV`G+z-;-^X4$5}5@b9=s$CpCXB#NE1;w^NK`$`bszSGZ?7cNVT{MAnNJ> z>~t=z>~VpL`3LaWxF!)i@QIclKZ?p5&#{N9lZpC(3w8XY09KZbs8SUTdaiZ&76oBL z_*bKH-8pRZxH-sVzN0yZwFp-yipZPqgu>PHKs?_QWTQRcmDwij?Qcbc4S6WHITLit zdhpUHz^#t^w4yc<+E)k?qyLoHd!ir6v6@AQyvfkMtqCh1Ofg4Gw{T)i21vMrJ?1X( zq6yt+hzhADFH7}+9L&NJNiLDho}s6uEv{Uj;U!02v_jByJ~ei71W(~}7-Y5p{I8Z{ z=dw5i?YU49{x4}dzY#t*9i*p!Um+Edyrj?0frfO*Lw|ZF76-||Tl?dfmn6Z$6f)mgDgd4ald8&|awtVf%NW>j7)5|6WF)_V&@Gt~pS=`wWqu z(8LfaVLU&ai3vSU=vC#&-nicm>bG^V-bfZn9mVsyQeg2m4zk4-K+JD77#DDbSCWs( zzfLRBkE6~!x|E6!^!?%BuRxHV2nEZU^GwDN13nrDVB&r?#@a_SJRA{hwC)5(;x1b~ zF9Es=Jjs^|aR^`Bi)KxqFi*JB)3NGG zcr2vOXo+6OLocdGYlAn&KGmV(whq{OT?jHPMrcWHEj5a4BTvF^Glj}I zp9L1-zmK!%P)#FFWaqF~wkko~+d#0kb77~bK z`*IL;b?D$S8BR&+qSb%> zbkm3y#5N|QOUYl@qNR-IUEjc$NeA%x>oK)?H@$! zI0pqJFJSxmJ0!>HCA(zqFXKrQ3rH|%!?pLHaR2tof>+wb^y~IHBq-Q})TD1izO8!j z{&6+^EVBlLqj_*lE(&7elu&W~B9^1)fw{5vIH)^9d|4hC{H+YT^x{eXR0uutgeCe8 zd^i_fs9=>W6>f>b?m6E0ATEV`%aMgDgAh8#YmJd2)p(;|C#VlBgY9ILn|TzNV!d>F z_D37;TQ7p&H`bB|?*_@8a}f~IybyFp8}UQUVHndtOV^zKO#5AJK<4lk{Nub4{;Xex zy>DL8wPh>OUNs23Kg@c}R+q|lxx#}CYxueAJ9%z!4d=#v=7h97K|}h3Y)s3hmvArB zZkxb%T@WCOx^AR)NEjLZE6{g!oWkO@bnTgcsLrxmXzq{(1}zP!^k+QkIJ%Ls<4MFO z!4Iz%-k@;gEEy2WphbdtG^tOKn6A_Twd zeqWG<@)kRBENm0a%y}~zONdH z9`iS8a&F)(i2MV;JbX`w`sU*PhBstka2c1w3F1VGr=W+l2IMO1Lm5xLyUUqz)sRD@k2aI9#~S(g!zGamBDT zePwNi%f}ng#c>OC)!9;}vU@{tivbeFCA%=XxSr6tJ5be49BLNY;hzuh14;`#~*#(m1FkMGg=euMjf0I;8^uCXghM9uBq5>B$dib zU)N^AKW@n=?u9sdMGHjg*5b9i*F@ThLjum6!?EA~P-f(c2ZRKmIDRQiI(kEjS~A+3 zK1TT++0eT(mCkq!UEVePjWx=cOHaMd#`__E;I41ysS#bo-jjSyddJ?77pEMlm_-fc znv~(qqbs0sWQrbbse{hlek6B~7(A|PL}k7m^mF$EZjjeHaBy`6$-f9jEaZpbubDXQ zx0JEv6MAGE1?C2@f(ECC)H4ng!Kyx(VJ&pER z%IF~ViTH#kgV|sNnDI;Dr_}}^Z#TmHjF$vs{zw?Q$Va9gHk0q;&X{nel&q%PU{q;{ z2w#3huXnIQW=gTzl^>4%2)_ky$G$T3r*U9AA6Ur;O2$Z64vt343sA($)lRj?P z^*$PI+I2IRk zDF!%{;h`c8*gd$4F?2}*9g9eig+ZDelgG@BI}9E21z0A`gAKutjP+L};fJyx#Mou_ z7e}VSJ*8-<(@=&_59LT(hb$&Y*5UA_jp%Iqi@fi2NB_)Zl3*uJ*7s_H%bQE&twjcQ zX5?UrPzm90&tQUgJ}1wj15naao~}`xplVk|XZzb`JTWfE{5fm`^J^T?_?Hf>FWwB_ z6W7urz8tuxSOmk&Td+BnfD+ogtmNI_ScSD;ORDUC?Y z!Npne@Uka`*avvACcS5A=fZ#JuLA}+DD6grE9z;Mk_35Twg!bN=ir4Ms&K^QD49$e zqgnOYfO+fCHMJbtJrY2-|1bPpBn{t2O5pDMix?wUN0j|^xCX98RKN8yTNE6No^6Zq z{nA|68)1nlGr2&XvZ#0KB6XN>2irR>)ZpfRkRN?avlI((hjl7MwlAmwvAr; z%?n#BZnFM+V&QeZ0|fN1qr;(QP<_@2`MdvPcimPaX=`}c!^eE_b3`>dl*VDxZD%-^ zI1l!wDZrK!yb#sjK}}PtFz7-jO7YBgAm=ayvIMNW{9sh|5l!V*FrR1iv1`*EIwF!y zOYeGNmrOHVy}}<0i&^rI$7;m)d$3174qET2!0vZpm@$)xr>{=TYL$S-%hGV>Mgb!& z<3M-=W$>ra0u)*#fQADn;Cn~`GnMMiI0#hYWT++bkF7;PwR5oNo-#bDsllAEZfe-A zfF|t^S?Ps=Wb{J{2})N2r^Vi=Y_(x_1H{nL^39~OSPM3c3X#*RW;3{`iwHC7oJkV^^H@F^irU%F_*9GulzBEKs$+0aH25epa2=NT@M5b~9 z+!D70?cIrxb)Xuf+MOV4NSrG1onR{X%uz~&AAZ!NcAeo?>j0#3*5UaOz=e^?w3yp0ou5r<2d(X0zXlQHxdO@PUG2Q=f0NAJ=C*tb96&U8e27=i@&~XyMs91Nj@81LwA|8;@yB|M`9f6LV0u(eJqAz=T$&OQJ zshUU{ih8GDz+62{=f*%(qA^Zc$Dx?iZR$R40$M%AxU_2}F3GY33(Hkt_&^=f_*C)X zyLnhIvx14Y5d@P5(qKLq2hW~IbG$>faOGeaWc95hPghDH{5MVA4_zj?5Bc$buY9|+ literal 0 HcmV?d00001 diff --git a/test/TensorFlowNET.Keras.UnitTest/Assets/simple_model_from_auto_compile/saved_model.pb b/test/TensorFlowNET.Keras.UnitTest/Assets/simple_model_from_auto_compile/saved_model.pb new file mode 100644 index 0000000000000000000000000000000000000000..771a58c620f4fb8c07d9608f6f43d37aa5bb5e8f GIT binary patch literal 66811 zcmeHwYj7Odbsl;Kz%<7a%dbAP8_sV#(o>z%+mvV6@s@0_HAv#pP1qW4Y2UJHwd< zILpBdGBY4|^+SmjN3yAsO0r|c+A2Q^RjF0>s(B?oNk*XWVXm~NB>D>V4|bKrFlawWuX;&g1Roa* zxw<0P?&o$D(tEE`DbdK;o}U#2_&mhci@DmRLV}9(|*!> z?aEsl*%27he&PQ&uDk`S(6>=3*XnfC{EXg?QJoypUsuYtO4ZRYxCr8F1*Kds)<0mR z7t0!nc6tX)df{(E{~Njb8+#?v4?+$b!~Wi5EgXW_ZnaX_%d1VCg28$b&dOus5Ktb>!nH9OCSK)6PtNa!n;?xsPNiRd*LUL~#=OGzrxH(B5n$F2FU2T}KhA-SPokCAVV=lE{AH*QyBuQKeikA4G`&Orl?gkxeC6@Y5ICSYKETm!Wr4DeaMA z5Vw?EeGiq5iOCoY=JNI8L$!0*oy2idy0YUiDh+GeFfcNXiZBSDh1jN|H5heB7h;qY z6m0cwqW`A)*Uo*8-xTx`n#R${Vl<0ADWPyd1qQJ4dZmi8h2r0@NfFA>7z}BjlFPXr zRjMXoO#P0WD;MNishDSX={Ur+ay6-6H5|1>9Q0oVO2u0JEVCcym=G|gl%W4c?qQ2x zBh7y8CHxvne7=sIFp#g53q__8abP;jw3Pi-Sf|*mlf^r@0;4zX=c)yz@CvE}V+Avt z_i{D$H}VN(4Y^V+#G|MUP~O{N*0EZs#iM8sM5aURrFab1AaryhVDq_R4uI(2uOzDsG!P5g|PZ3Pajmv?rB% z#rA9(hL!h~{2t1Y%zB%Dh@JkMsya=wCKU6V%C_?UD|@99TG9M{PFE?4pQfvENMJ|r z=IV8&S~luJ1&15zAn-bxKkUzb)l{PlGRtGV8%x*?&lC*Ql-*o4hr}6mQEU)=Lx4fl z?kgzHY&c}yWetaDrJ20cbH%b^do6Jc23Zd)1ywQl8XY4`0xblVL~iJMdJ0BW3yZH7 zl!r`z$dIZa+SqoEMu1t_L|;v*K2(zZ(#2Hj;-WPDQ2TMAnwKu5RxWI(rM1P?)S9%C zF02aV%Re2((Y-HU*(=ux-=O4C7?BO*$LK&~4v}n6a?lvrBZ~sN(1>JDj@sK6BU-ia z793OSUtktd{U2u*9cGeh?}Yj~BfMUz=TMBfo(T175aDXIjBxcwIKtJUp}r3HAf|Fe zPPI#V+|Kg>VpNQVe_H=WDDsW>LN`Ed?vrSMUNLKc{xe(y^upm9unR-zwxMs8T3{|_ z3*BH0prk+BOClm9XY5@uLTq1vBFyn$4AmxE4M}vO9U6&7UjI0+mGC}{7@vB8Hc)9r zK_(2%hOCIqE2vGJwKs8$IKG4KZNmQLP|bAIl04yRD0;LU8Melj0Mt)*s2>Z-X?r7w z2vqI{p)jtDeT}JimQ0>!X zRDUBm>*~5;uu)*^yT|C7u;GeL7z)Z(Zm(3A)p42G70#(oF1V_lf$8iqGARDU_(#A7 z9@qa<2PV~@By5yAZOC$?SKu!r{(h3514aGFr#7}wP6((&+VzsUAxe&EPS*g$q&{I1 z|49z(9#KG5%d6(&$&2-Q{?QplT(nK2P)&7 zrXW|!r4R6zHkH@dUn6K)XfK;dBriBTw;3`GF^FzlzcnVXAFtw%9x3vA&#LeRp(lBs zd4ltsry(I@U0X_3Dd%y{kFf)F4kJIR=5}|H-j&rw35K*8A06 zg68RD@nhM=For#r{3WbJomG;r&aQuXR*aFJ^`DuAZ%1HI6iBR;`#`B)Sj3BnaglJ-Lp^FzgDIqE z0w?x3xyC4pY&|$8isVRD+1@K*+0SV+?^2~!Q)*bn02x3#kgLhL-69N$Ktx&2qcbSW zkQ4{Wy!xBXtEvjl6U#DtWKLIdC0Va%S{x!%rk>Ss9%c0ljEKVo%x~dDe2d& zG>GCfhD6YvP`RfPdfZOvj31#V{0W`46FO%j^kfr+COAS*afGT}eA-Uv89zej{Rw@} zPUu-1q36O9id{SheaKZ|5&ujRA%%YuM3C^$DIzSXybUhFWcT+XFfAs@R2joqUP=ez zdD5@yeGRqtc@o=EaEh3(!K`?JOlthqH)2jT*|nR@Io>!mXOu#y(QxYdvDpeVDGIM7wo;XalzhW z-Gq@yF+)Zjj)4A<6=45;o%Q8vqrP0@`ZCM+5)%!?DDx~m69 zk(-}1xVeZ-VmDsoZxFjCsa@=Kk8$`H7>8eK#Nn4Y4qxIqydK2i4L66|HsM2Jt2TRH zJQj+>Yz$;9{VATM?8asO#yjLTjNHDo`P!xRtC!{5m#*HrtWW)Pr0W~u@K)cGUEZWtSDI3mvrfj^@NH$*O zWaA1i8?Ob)#_Mj`XnWKg5?dW6=f#sv$;PL6QDCxhmA`S9+;R1u)BpF9j#NnL_zbz} zs^1ngpkDH!#XN%K!?3)j^?jpJ|6k+!|4qLCuLt%2r``SEwlgv$w&;DJ^*tA=|21Q~ z$#*vE{TuuZ#<}NEy#!6mBAPFQOGM0<=|x`#|MU@I*;KuoTESa}I`UocwEbR1H~cp5 zG!Bbz@ebgCc!y}=Sr`!C)?>kHORwOYpRvFBS^JysSZ}^-M3G0ty9DH|vi#r?Te2{D zc1Ppc+Xl}j9CWD#Z;>}$dJ@wEhIHASTrKF%+`{65`Y=dEJge2CuHMyZf$$KP`5moU z@AA#!nWi@D26@BPERE-0Ur&W=lxL+X-QJakY7&!P>;g9Ftg36n>SCBJ#uH+}A!du) z!#8{3n9s@<29SJXHV!JN7MC?FuE!z;4T$;C5J6Ac*Liadqf9Jd)nt5Wp2e3=ZqC9CA75hAyF9Lp zsX>jEwag0q?})!_Na1Cp#V{T>UAynl%W3zW6WD6wlrN{aIVFVI&M(C`N<(oR`mF9 zbK&5#u5j>Cn9LG;IG8ODBs;FDuQRE+ytx6FcuCM(--IlrQ_J-KMW8Y7{WW^dhu>Lk zaXY)H2j^AMvVKdc$-qa|N=b(ckjPmfIq4b+(ECcv zSYsR+H;}Am6O(7*2wO=)h!Nsa85)#Tth`?*qh$EOWw!9RiNK-aj*?8nu)cJw0)lGf z44i7PK%4E+{*_(~GVG1*zO)`}or26)_ARl8+<<1+m~ok6Qr^R;wOrd(^2M#90qR0< zsOGr9KkJ2eJ23q5u8`d;p||ZzrVS7V5guU>MSosjCY@X-xcteyrAR_uO&^Q)^w4)L z*rG=$nEQ)^c{PbLX9%>y47gzKN#cQy9c^ zBCouQhcF2r0{l;*J+EwaZH@0sl7bVs$zf-&q~bXYOhIMW1a-J4Oc!95EfM4j3^R}` zc)w%LqU<(Doe3RX$JcSch8oDYZSD%$7mjG#8yxrP9(X?>PW)#;Q=)VS_L1TxSANm`UXba4ilpft$j1b1{yl1H7u)dCc*j53GcpYqpjEV zy$R|L1v;kJ?_st>Do%msJO>2Ac0gpg{SMfg7QLA=JL$7k`{gC(ML|Di+(!lt-9IkXdACW=s`b(as2yzunzNlxWHqi!zF-~HED@A zfp!B$e>(!sUK#OL^*e{W#OblNNz6oD$}8!Wj9-YsXFhMLTC0N{piZlpS`ArC z<9B_&^fZiWtws){=Hc2;_Z%&~vdr5K$Dr1oqY>{(SLf$woe0~kW^9u*M`w1!HpaG% zp9t#il+L*bPfgh7k7Do(g58_-2>h$w?4m6=LyqxIgYW3i`wUHu{%j$^XFTv<$!Re9 zcUNd?ILhM+jn6`Y53fI7qkPgicRdDE?rlj0b?Vy6$lg?-t`K4_+5r>;dVc(X&Y>96 z(yDttV4)b&OT2R1Q4Hzj#wdpLau>zmx2vrMypi_`?o*6nK+!T$3@AqjW1PDvhB}5O zEffPPsqHF#Fx}8aF=%^ao5M(j3{pd~i(=5ccDB_Tx+n%w_;{cgLdQPsYs}r)Cng#? zwEOrw6Z=FQZ+BMo1W;40%#fZK%Bcg81_YYgW=dU11D~~p&zaJNG%!!Q3u$1x8@iAN zBN^?}3u*W@5xyU0;W(XM#cYf*J^N8ycuKve|Ite9*l1OE>A1;lP~D~DCj&Zf#eKSi zXs4UagTznzfW$FVqps`lTQ1a-M|e>Mmt-2Do_^<`P*0%|Uk4R1@nBaFCIk1Bc|#6E zz@9?4{~27=d`#dQ?e6W&U~)fxq~kVst~B1F(}MT(bTMc6X-vbrICb?z)JM9q8`veQ z!GnhR?(R+_gTcplcjuFH+<)Z=d^c*}%hfD6conDpe4>jCoYp{MJ5BEBT7om$i}%9s z>msY5ZP1rfm`6BZ2bmp*wgS%?V7Bld^g^i}SF2g@-i)&^Olyxlx8&>#T3#O98147E zws{x}cve8~!m#16se4-W#KR3XU^Takb_Yg{`t_}?#I>9Ds8OQ*U9~?gKqG#+Hll?R zwI+AmeMvAG&m!(INvHsL1b z9=)S{i@BY8T8H>Za~rI9$H1Zu7tIy4MY(5P5uJK%+7dbmv+pB z-qDs2#*HT$vp6v{x~J2E2y_cZbuVt=hJKupx#8TTr)`l0df#(P+cC$tt*z*-Z?yTI zJ(tIfp3B2~K%&R>$KJ!yteNP;=XE_(h@YMv=G{Cm1X$~awp_|K9{3uKiqgx0iW=Z% zm^n=R`h_%hL!R8Oq)tTZRx`fQy39%=5OXGz;r1umLCl#|25`2!l(tlCt3i7k8>fW{ zk&3Mwt&??+teJF42bPO1vtmW-GMR%36LnF{b`&#eDTiXttV##d4P6v-7sbrZ)9vPB zLoQeovyVqEwkH>@`|C3-A6sw=TTJ854qi16J|>GWlT%N>=F%i-b0_WGAGZ^9-s6oF zp1j0nV)rmUXY-4e;H0a1NiUp|%}>~!B~0MtdTGgg$?**!m}Ascd|B2Lpd&TCm=w3i zv(yTv{KFJ~GFy4oN+lrfsw=J4)v!!PtH~&X$IXHfoR-((F*U7}>zMI5=!-nepk<5Sn)eC^sz^<_piEjoEwHwPwq$|~S&h*`*amfxx?quxB6 zW+$-+{g<3F&j(kNr<)1u>1RD-J`*N!S^cE(?4pwxGY?_hMHQxBW)#ui6ToGgi0ck* zJWx<~hE*pi^Eg1=$CLi>SjRogA%pp3)I6>QWfxDG$D1+M&|7grZ_m!*T|%W`ABEVsay-}mCnvIk!pcyJs`mJgaGD~(yQ z(kYg#gkp*5u5v6{IcS!o8?z+c0!x0^OG?rnEb;ZtIfkSUnjx9S49RqgA(>DN(I+At zLox@!khl4iY|a+9HNw%q9pT3MRgT#kk$uQydAHFvL|4XnRN1#Ih znpgo^OW6Tyiy@MayFyniJ{2zaxEEy1XS1Jg?(w=`UgKtSkDqW&4~GD<=3!Fy>DPg! z9*2WTJ??;(d^D-Y@u(!vS-vX;dFz0V!=EO+nN({}K1+DV6SIZgQJYy=u_G6r&Roph z9ksa9QmI$S4CcL#L3HE#tucZ9colzedjh%%N&EwUo!e-5kja>z+)`gBfceXIKc@@K z&lUJ{UK&B_y1;xRyRnwsn0A5rR^qq?W8!U%jfp&rU3=@HeMY)do=>qeOJqcZ4 zeoKdJc7gewIiRx(%zpw0gZ}3k_+B*N2z%lJqMe85*a7zs4Rn}>4O36>a~$J``J>Lk z8Czi%bMq;B0nXWq8mKd$;!ZMBDyGe+@H`~+x^=W!U^yLc=2Px8%-QN=#}}$aEpQ$- z-YTWl=3QzGhV;4+XHl!iV>V1$O2?#;HR*L0%ZC{98ui}r}5~g6A zWcmGz`Aub8dH?|GANi<71i40sb+5)&y$9~*?9eb5RSdedij9#3Oz^NUs8v_jY6iM^y_oV~?>0;lO$fPwDN+3Igt-J92~7 zq!eC3m})t7jUd$*AK)#Cd~GSPV4zLr*wf)9ACFPgvYAZ`wQ0 zmu&iJFjS8iKPYv4FU#<|j4^nSr{}TP*!U>osTjI)80bIgg^3$zS8q#QFz40F^6g7k zZ(Z(!IX7QZQJWe!8iP6i7M`c0$xn?PUw!aE&ad^NF;yu*W2*jbq!aYGhqRlDNauXo zVwDK%h0t&w2o3LjQ2VF-Jr5ef>oChS@Hl9ow_%p>^)#r>%WxXHwvZqGXxOje(Nk{6 zWg%}mG5Y=n#FO@3&*dL96qwjuO(vYQ;|R+}KEc9K_(RnS;sd!iS&YbkN#cLdRO%N>yFtL4k;;x%;ClP^e$)`Ce3lH3NqlX=W0i1Q;1)S9C)xCV7sQY%^3uOra)Dum8!9O4|$zmCDb5FjDT z#qyR?#VvG-Tq|yubM?KdB0s9;c6U*SS5_CL!vGfhH?63=k!t#5bo|0u5VpvvP4v~&24h*(G1^+4(U_vWqq!d+Gf{cX~|M_-v$y+}MN9+A zgkz1*xRo&%H#k&=YAJ4KRlqY%>>i;DJC6K@XSqxE1zb!&rD_hF1@ z6qgLvMwQM$v-^zbu8<6QQf`4_^f`&Q+t*xr*k2Z4l54%0tI;XMTF-bO(lmk5 z`o?_QA!RLh4zOIUp=_j!XU*CB2BPfP;qz;PVE|Q@SVr3tW?k=%=Pq&9y3=Htw`R7% zv#LR%Be5$nlgc_MY+JN}mOWPiW;M}r(9wwmv;WDSVD{8qPKbR*WSX!Q@*YIb@MWyd zbqF*)?I`A1SsP0RGawpSpgx)-3a*42Jwl8y(ko%c?BHQtx*F0DgfbV>>)|YM7;~L{ zVorLZPy8Z`4IxrytcL5?RoOytkZA@BQnvO=*AbdjWRXavu)$((DLe@DnBPNbHIkTx z309ji-`mL*%W`Fxh=5sr*rx8}i*Q7i`69(~vCfK|A*cKvf%YzC%bwpxh)hkXK12u+ z=3d*%?Um}}R|MEFPq)Go7LGXzU>st15A&&I7smv8%pN+4h{tj9i~T2%Pe4q!CYXTE zubFBk1E^PnDMW=$)T`Jn zxqFy!&w6l76v>gQvb|TzRg0fPbREu8OO;wpsbLiZWB`GW2xiRf7GX$4Oq{I6U}Z>( zgJfQf$8rb5Fj2`@$^}()FsG7U(X=>3ri@-wg}{mn!H76azzh@B631X%u5YnqeE8k6 zQh!vbJ|LJ54|l4~;^wO()at+uKcS?VbqR?!**~ zTGaA%ewWcXX{YmuADu`2>726D8Mo1SEG(S>A|?IE@Ul_@QJf~d_i)iCfcoQh>Sz3@ zKjBaPteyHf8}%ofpgzG-e=3N2RYp$RsXyaK{k%W*&)KOzYoq>LSnB<``~vi0w+M^) zXPO8p{F5Mpgnv#EVJUF}j-oc_AE??~D%R>od{x(k_THWgtp5iQm==>{s*L8s%dCKS zp7g6CUqid}Jc;cn2>;C2U{<_9rZto2ZFR=Qi)2_UV7$`-TV^+24GmH=mTexbV8Uqy z-4-x$9MeW@D`f>)An)NQAy@9b$MCqCUHj)z^MZJ)znIuQc{?@*kstkkz6(1W2vgi%)? zJRe zokYJC?HLo2_~dhZhv0?PA$VbR2wqqn0>9sj9kgV05bMWfPEdNq^ez z!B)$wZJ&(QUIm#r-Bsx@w;%JL)CCxS+*7u+LN+urAp*rK}$^WsDY2BZIu zawUG!uuhB8zeaQ`KgxKVWMLeby)VEwQ8ItgA(=C}WFGl2O59ezp@r|HgqnukY8?Ds zHamOKYV3b=ByI9&EH~Y85A?XYomnG?gb!A`;iBEy40QjREg!7Ns+zvkNKIen)btXs zrt3j!y5UySwkKahVym;QdGS~W)D$IkQ2Z%NaxuZQ%^B1J#hM>&gH0V0;k23te}^gQ zWvi+GRj@l7uBHa5=-Q!CQLEK3o1L+(e8&Z8NNu^EoF36ZN3nnk>BeWEJ!3O@Rl9NW z$flAwc}#`U?q=h2{}h49?_d_ zs5s5WbrlCw{c$}>Rq|jZ>_iE5n0}*?B46Va`AuGtuLmjer`?L&cARcVY%ws0-t@T+ zDDwYh!}O*l+dp)8)B3uDOOd4|f_bQa1HEaKW7|CMjP6a(d>Ex}Z#q=mWP+Rb#rC7U zor{@RErp+FqxKD}$)3h2)vCCf^s<|1{iYA+FhfPk^tWPyBNZpV+qc;aD*|h;(;CX8BkY^^kRgHV3s?K99x) zSUwgtJ!I`ZJmUA~!uo%447?r@Fjf38_Gfj&p^E~-7b5FF=}4^aFf-vA0(@`tAk$&- zEgmpBAl@N;YMNFU5Z@+!YDTj}9L6lbu!erru`+kDzR%d}`>egbcdYu})$2oS^<4ri zVTK0yX;42~(7&Ta4BrkDF}&btUjV{!JUfa}&7QgFSwYg9ZM&s5UtOnsM>N{<{(40x%e2BK584MFdot zI}=Vmj-0Tr+#3-v74L64)ULH5AWtGl_`-hlc{sBlO~Bma$PDPZNL=4781z2BA3X<& z$B|hO^|PxN-KBBa3RHmF7aX?GgH&TEzS9~#8o&0+56YECWlu`TKkR_NRv-~NO&9m0 z7hrxrdJ5Ff8rI}Yjcm^TJE7z`VA zpJw)>%Wz>onuN2DBWFy>3f2hqol}n^bDV_vaG3GzOAZn9pqGi*YG#SN+d5II#K#OV z^CXA-V~3nI8#QI|u!p)qd3itjNl5QUmzpssIxbo>EX^ru+q7fcs!iGjwm#Vk?SI%N z-&wSJPv`qf`_UI*Wj`u4A+>F=y=~Gls@xvwEpr`{S=(%tOWK*%;kE>V+q%>-vVD;b zuwFRokWf^yC7U*>;@B2~Drd4YPhPQw!}mHRh8-gcSM}N>syR;mdbn8e=XE}9>_=aQ z=zg>Z*1i=@!?NXhh4;fc;evV0@d5_PHDqj_F6~Db;oLrMlWFXJF^8%}>*#dBJV;et OFr1FwX^oy>S^NKhwF52y literal 0 HcmV?d00001 diff --git a/test/TensorFlowNET.Keras.UnitTest/Assets/simple_model_from_auto_compile/variables/variables.data-00000-of-00001 b/test/TensorFlowNET.Keras.UnitTest/Assets/simple_model_from_auto_compile/variables/variables.data-00000-of-00001 new file mode 100644 index 0000000000000000000000000000000000000000..0061f386553ed3e78c921275490ccd293f634d96 GIT binary patch literal 322030 zcmWifiC;}$7sjt9l~PhEl2npPqCs`{YLXO5B}r13k|Mb!B*|@_2hDT0IfNusXRm}L zgpl+jWKO0KGQaoz1J37j&OUqZ^{nUlE~z84FjDlQCrz=pFe=xwn>pL2m&DaSfnqcoz;>YhdnFQ|#xr(2TT1 z*c+;X6Ms(<&6f%I*B}UkK3YKT^;&EV-$YWT{-jpVs!?+^jvBS^hl)8>`0gbaT$j~> zVsBuu<(E%q>SbuaE@+%i&aFi-GAIyXC(nS!|c9|s2d_q`(lWBaaCqBzD z#^Ht}5VKW;ts9o&5aS2MKNmuscRsMUOJQ956Vh47Hs+)o5$utSIJjW7_0;K7p$>DY!UWlH8IcZOQFQQ02e4U z;BR&knwHnY`1E|r(>@f(a9LSqdKa)*(xFClmI(n0oAqgkGo9L^VnSwoFulcDE@u zRjz@*ZWY+-^qv&zt_M~sFi|U6==VSs?oSm)0}=u^%MPL9`VN|HxgF(w(}ABjg9%$I z4`j|Z@LydC9y~SLlWGIDfq|gaUrmE0q;Q-YO+>E8lk6{3VSF@!6rMOlV(!MnpEoRI z62y^nb}a@QHNe=%Syal`3xZ=4klPbRhs@5<;a{S}qd^*T1JmK!b5ZCcJ0bG)E-?Dy z1?r`lFd-98#_T1DW2XzKt++%FuB*d@{v7OE=K{ll!6Y)OnDz)C!2aYXv~T}M#{F

&El*q3yQyzGcZwUWtw9Hv(K`D_C^9rCor zkoS8PSnWI79vvb$XVWHQVS5%$N)#ogLc3^Bg$T*=Xe7r^<)Zz7E1husN~&}@$UpU% z-T7v)U1<0{mHnd!rAL1dR%kgi_k^LhCeoA*2@rR(6ol?1!Rkn3tTFV3lrO4ipmTvp zRk47zY_qx4OA83v{()#c$Oliu0!)8Wg--SH5dOIU{cClw-?AEicP@g$HJc!7p&!WQ z$^!T7VayQK0qa;R6wi)_G2z9u+)@!t?xw-mhqsL0R12u-i>3iX>!Gr_0H+piMb%yD zAT@m+SQ!^WaE%;vw&s|#T<4Sihgx`ZY&ptDNPv)47B+wJL3yT_P~m>E;Y%(WsP9In z`!48we1PV+c;fh=1*rAevbnyA#M@pCIhT~MxV{Xz{yg~b8IZnC0n<;1@Y?MP5I2mX zO(&8dK&c3|oqtl1xK6^Q`@lP7h#I^t$LCKHP(^(ma;I3q51u6!KC>gOYz^c+ztPV3 z^PwJ>wW!zK9CXX9B;E&WV8VPA*wi>a7;3=NTNJNF<|FS8gN>Hj)<+Mk@pTk(qcz4ah^>Y=OuybU@Pt6+W}uP0nb|4pw%f2 zT75_YAE}z5+~Pz~Q#OXvf~iar;y>vV2bD zBZ{fKUK4S=;{vqN0t#0~L(`5iMz1Col#0VaeP0i?JME9WE4g4b4xqQ#9(AtkLJw;i z%nvce2hZZ5_2zsCOBp7rqAT&2TMe2%C;{F}545@)4RLCg5VE8gvt*K)uY0qQ)wQNQ zW4biDz05`>kCUY51OZzn8G0Y{uyINbvGR2$+S~x*m16@>A`YT=U@1Oyo`xfzb^s^b zhe`F2z^>nwFlfGn+8_KxTL<#s+Vn~g*%gWfRhv_K>C>`Rq2e6&Rl31ty8v z;Hg)X8MJYOJ7pp0p|=C{gViANq6ZA|Dk17_Ha1<} zk3=~WvNP*x+~rsz^4}q7uC0Wg!d1{W-w(8+4r9KZE{y(5hI@ZYuqHDelM;VX5!+sJ z^MX3Cx=xvie?A0$6EQg9SVJ5C^MIt?O0;%g22N~EgEXI54EoNHfz~qUlX=gSHXS7g zesS^73nkPnW25*iH)IXmA;TB1vrWach=Fl6UbJqdvI(C_7XJi$UzZ-rC%MAjkgrrj zr3RN?)j+pUC+M)B459tGa5ORnFP3!BmJ5Y2I2Z}qGref%SR{^zMg#xS6E>^*C_77$ z&tCDw1)<3l-BMSAQvEd0*47}oMrmaDl?%8>Wk4rq2IG-Y3Y|M@p&_u7@T&Hj3u*7g z4=GW|n_fc-S1p6z61$s>q|2kQ*63v6xQ8OcQ*s8)jWTLiu0P*%5m-AumYBT5YUQv2h3;Dx#-4u4c20U}%w->v{W zzB_Q@cTqRl!`O3t4TgTJ0ms;Q@Gkj58cH$nHBIJdiU|5@;n(^w6(DBCg&{|kT zvY#!4cQ3o?-2>UECs_Z%y>X-=V=IKujl`z?`RowqWVC;`1MOxcVfk?lc%YMsLZ8eq zCM6q#;`YN}qd(n6%aHZ&CX?Iyk6PV$#Lf!#1bt;;?DLC-PIXzd`KXGDX64W|Rsf2u z1TYozfgx*q@c6PD^vpGYH_rpvb@j+P3Z${IoG7R0quBF27%QuzvQq*`^ZO#GyybwL z8dcgQvL6jP3fY=#^RdNohqj((m(@e?;66MnW{Bg7Eu`;S1=cw9lh&vH82&dGO510k2pEA?cQA0?M59NJ z3yq1H2icu@5a)i2@Q;tOS$WGL>A5YqvKIj7lohjAHxrj#KZMTqS1G^i1l!7WHe-6b zkb3++Oh^1?le^Rgd#5DP@w6gn(aZtbTt{k#s)3t*n_VRBg&RNXqrJQ`kzKk^c0WX^Mz6zl$4@&j`i^F+eYB7)u-)7Mfk=q8%J&G!2VY@Tyc-bu3Br5 z`K5uo=S;>Ouk0}_W*hYGFBbG4m&|39?2wo5+3r+#oW8wMh^m|wn6Ekq;MVF68HG;xAW9SaXD-L$=W)2ct_~+` znyDQ76%9Au2X{twF=v++wrWVg(3=CmleB}r2trp1bhZ1!d?d#8)GGZtb?OR5`F|mx zzS08yRp$fUTL%-bSCMnR(wHf{0*BXC5>}_7X>(i}luCrbA&p(g6V)}}GV2@7TK|SQ zBw~+7+Zv!`u{81odTH83Dr&|ZM%H-0xxv0tcFaC8IPfkX_3wtF5*EPD7n0Z)Spf#e zec*Gt5{S>thrX}DFxAEmZ5JfMFVAQ!4=4qXS6tv$yVJ%S^`K%<4ORo;%oScbwzi)n z!+&hpp^?);rA!E>-tfUz&v?q+?T39<^U-RNI~`K8g*Ly1=3XyNQV;8>JE4cu+#!W@QDvGAz_p4#Bq8K{&_(97qK_4^64dhRyf~uD? z3_Jf|SntlY_5{h2B%1@^<%4v)-B(&_8-QYQiSXd}UgR$mCgjXQFq*ytyw-<8%klxD zFkp=$hj+r*vNsF~|47mebFgPX0U~+7h*Ux-1YOUCnIEDtEv1a;K25|eU#Ek<;{e^e z`xVW|E`%+5!DzL$gWdS}KIyr_A>3j?ZQ@D5*N27JWR?mIg)2b)!*t{-IAKFl5e*Cr=3a^TS~zDf;fsOUZqjG31Bd#m`0`VM*HQ4kacDcRb}M^ z3Clz+%zST2V#`WNj)m&VXlZQU9XF`iq8!o@Q3C%QOLF>+1 z@Q^4c?hTvp_pzs>Fe43?t0?0ACn1cSS+Vp2e8k6ho84V$eA$5$evCVZwklEG600W40Ak9HAhZ!H0tx?WF!o4;jo+ zB^7ENp!e|@+})OsdhH{$wPGb2efdh5`K6E(REs}OPsegW9Z`ttC0WnHP}Tap+2vcm zm`zh0A?!~n^j$B6`gyvPl-?ly@44{p`f)n8Rg|LtMOa{IfS;b0v7Z`hBhjA+xqTm~ zMB;2xII@L~ZAnE-_h4Kf^p{v(-$Qr)>VQPEJka3z!{)gTDC9Ln_2LWR%#@`d^>h~e z{1cC+Tf}fpT84jjH!^O}%B1!gc>Z0xQHlgQh8Mx$YfOo|61&w|>KJ1W&l1DcgUB?uV&%SK_ z)|3x&J0DVXZ~^O;kznt;iCml230c=e;ANC8Xy020PFc^X(*NtaX9A<LR`na*<4PKw~%XS6h<_$O&xq-OuPQsw2)v(jY1>V}TphHs* zs^=atH|AEN?W<0bJL(AM55|);+bg6Yt{C1262@`kMto;x0bfBMy`GjryrK_Eu04pp zSN5TuY6~g(QHE9lMNG+_&xfUDDIH1EHU>CmjM%LY9Y~db?<;Mhm+tTS^YE(&f25vXM_2m{xI&uk* zUCIXU%#U=_`&@Xha-EJHi^LC=U36w|B97d##GcW4WV>+=2I=teXK5I|J+lNqeRHSV z>weN#%gwO2?G%ywTM4VDXQEgoh5p|S^hP#P;YV%6e1eN9H#|tP=mJQy+=YG@b0Pem zEA_p7lL=`|L={1ww>RJgUE01851hY4RvH@+HLq!C*eMTNUgu%Ovy((@NfOTBX0tn7 za^Zz{9)^iZLuOe&HFP))VfD9}w=-AZ=&dwVdg%vlKVtEL$aYja&IQdo)3DJ}o_(j4 z!;b%)1^Y8n;F8%bl9f=8(<;;Pz|0ZyyIB^zO*BdK^gwVuQHbXT(lO$57;fmY!#OwS zkfdYNsDI0EYQhwP5Pur@o_Wob-p!&$eN-hGUkY>)#wPX-nU{3yfesbsHxCf1$G!Y;?X zF#h@#RhF6pw%Jo)TJa_#dFnLz)xIB-WF*OX*QZ3RqZXX_H|Ql^7LMH+V0*2|hSLqj zxTtkGjdStBfPKeEUFmKty{mwV`?x5k_nj^BzzCvVog}{$^{DQiMez0wA4IGj@NbL+ zY@EmhZQ&8RO+5xnAKoE7m$qOC`yMrH`bWeI4#BdhZ{*FDc`(-ci*~USf!T5z&p0~a z#Y0cX1yOgnKW#I<{&kEj8O*?*L&@-HJRS2CN=WtA9n|rP3b<`3g0Yetq*G-YQR-hs zhvv=)*V_^>bda4Qghn235|w+Ks0ku9sI3auY4!Ro{T#*McYobQ*Tc)!3CS4?IG-y>1Ieh@XN zFh=ZnPgT6TDd%o7m77+N+6K`?J--3;gxTOJD};`$XjJ9T1-{p6#zKydwktQ`>^+ef zD#$Hj9-gHftw|8MCCcHi=jWYm0UV3$TE)Q0$<7xJc`)VHsw6kK50w%SKTmfb_!Ol?xrHc$4O(i5;#5o zOD!aHKu^37hTYoP&WGzDeA+@{z;>a_Zn6=sl z#CB}Pfi=(Rvek8Hqi}?Jl+FS*b0?C!rxwdU=Yo@B1vvYqf@|4f=$!P~d}87a&1%|( z5f_wzQ#*%wd?68*iZ@{7pfmL4sKK}FDsVEsN9TOd#-@c;pz9EeR^5qk>7OAsOH`uO z;4EfhhZ_uDUqZ6}bAy_NxlmlX5OaM6o~BziL}q;?L)Mpw$dpakT6B<%&n4jayAG16 zI?6oFM8}^8!SVHR;;gO@-lxNH+Iw9*vSB@rym3M9Y)=}RnF=+3i)qQWDX8XJ1u9#8 zpi47`$Z|^Q;>n??cc+QGx=;zFm$Z<6Os9&EQ(;ov9JHNXjjsx(qJK#a47Z$Nw@zZw z5tD4RosVXeRbbfuQMU0d-!f$UE%U z-Z(cF+?LITNbOqOe)1G;3VTRfykc>?cQXE1p^frshhXeh0k)=@LF2N6IAY;Pom4Y` zcxj=nWD^-@heJvd5A;>$3Ua#!G|ei7cBKrkiio94-0Gm!eHO{TAqs_Ox53P(*|2xU z6tFj0OG;ByiC(!RG#TG#2fXtkNtP_I_xO(#7v;j&ZA;P1^%Lp)RY&X!;?R41D_*pS z!w$WAOtM@G+?D>||49c5o%Dc5cbR*<%))X(#8#Nj zQpWcC?UZwQ4-xUqrlnVuL4Ml+oxOVjw!k!Wvpz{UynPUUAp^?Gqp(LJ1V(Nxgr19R zI#WIyxBeEx35R^(82w~-ehFkd@3R6~-#3KgFq-WA>@X#W*&9 zHgsKnMp(-e%=>PHP+EQnEij#cPUzXRe1DN2{VV z8ZbMT^3+U#$MG{?DC&#ScVkhcP!YLba_Q%*fhbiLf#PB@gy(vl?WnLBe@7Qka^4n{ z7Dmzxr4rQ4^2LdUCp5hJ5OyxpWx{A4<%AL#p$qAY(R`{}DbRoW!}P>EKEAhr)z&Z;iyt7kPGu6%JC46 zO$0*He~I+QunVT$9wMFQs$d@@O{G@uL+a-T*EZ&2kXR}V#sp)y_f$U^333M&r5d<+qy(~mC_n?X0^V0ndzQi|TTIg*hi&H| z%kuzmJ|xnSgh&`$#iy)IJ3z~98|u0iAJ2H^OR6eyi5fe2L@@Gx?r z@5Wz}!#dlL_b(H=EVh8Yb24xuTnM>g21oMCu{B~bH8_z)yzvb_sBMVsk()L>YaGcO@ zB8wdA&{D1d$5))C^3#5h;YXX!Oi{xFp{ z1$-q|VDa|~AQ{f!q_7Obdz=VIXCL&0tc0*2(Fz`gDbKZ4UydH+gu z-6A-fHws~oJ|E{q)T7!zPY~MPN*zPYAPDur+0dU{|7U@5!uH@(Q-f;O6(H&Dho`US zVd;YFbT}@S%}NZXMt*$wJNY!VzrCKCN2WmU`ve-{n1$X}Z)xAfz0fuMnuNy_$eL9| zZ%A&(!u;D*eYGv>iN@nZ)&b~|3nN*x?vT*7XtbXeLt0csVQp&_23MS&hjE5UJx8Vqy)L3*}$ zgL=?hc;%o34)JM7OiQVWSqe_L2=?aee7f@7SNbqw0h*lJgV}GY1^IhCRngcEW9kdh zKe>iRKDa|h9!G%q>RK>xdBIF`*Ta5!X()U>jW+d%lH~FwXtOg7`0qaw*VnONb07m7 zByxba-JVYM>88UiTq?IF7ypiD0+p|)2H!P6YWNLRQqMeIZM!ggu3VkC=_1$iK$ActXUj>!74l{ohwB^PF^VJ#zxhJrfZ#qoxlLy{=+ z;sI%i=%mAAiJ-V%1z2g9*;o9gqW!obaeOuxy*%tt{aqboi5Za|<^l0uA`IpkHOLMB z!}KXv62E7Ykt3yteLv#a?{ZS$u1hpFe%JzilMJcJ^%|J%V2XaLAJV3yF_2ZV1>{`$ zFi?^UN@u;`_qW4@=a@kx`E`_cyQDqn%5LB){3e=pM~SUi2)u2w!3RBn_D5I1FP9kP z#pVjK_9EcVG@t>ClZZG=uy;O9B4^&FVB9@X{Atop>HSCK=ksV7-N1v9f@P@khyxs5 zHClLXKBQf*1a(U}U=5#dn|V4PHa66t^UPS{l&=WBqwA2$32N&fQ84$b#FX>3c-cJ= z%jNY^%3y%7wuutiGaG4QPYqf;y+E%&%)x=~COZ1b9kpun!RD2Yz(bz}zdbWSN!5|6 zd`<*E87-i?g52~1Lv^~mQSXX7)M#i!+GS}pDq4c=_L&g0Kmjj)tAKVNJ#4j|PWN>3 z@jz)gHVmY|xPBgV?tD!}QtlJ&s$gn1trpp0$|$6m0lbqH<~k}#FlM!vwupY9H0wMW zyONH!Zzsdp+U?laYK(jtb>dp}5&zoR6owtD6?IP^xmq3=! zc5_?Neqy`zI4SQF^x2ZpNT$}46FRA&WBZNPgg+)-^=Y_H@NN%E1>ZR)f`x2*P!M&* zIme4o(@YnpzJEw}&PYeI4L><6((uhznX!cFLBl1W?AeQ+?O zm~tEsFiowm87tRWG+@zV(lFySRWkoe>W<`N=T#Tz+-1)0RZk~wj>({OKMy+6g>ZtS zP8$DfA%jiX#I7z0Y>o6F?Xxd5?OY6FY2{4I!ZXxDZjj1f0kE1ZPMW?X5|w|ef%~yCejq@#jgQ&vqPonuHUpS}0v4$o-z|qk7|^z%mUoH)y#) zj-KM8efn5Ok)Q7hS$AKsgWlXHw+3afF|vRR z>BN9``d$)qRSic%EI_t%4Si`c35`5&kVY(nrs;+-eCRPV_vCDx6F47bmpg!L^G7Ov zeh)E{@WGZ@)^Os1G?vQPqJAesdrzdn@2iS1T>g!44jzW=h9V4?uOK>$^nn-6qQl4Y zn7!YcsI1mOC_S5tRu7umtrBGjXZ1b;?PgHx$uXeIH@Ji9Iiw>LBZ zXVWa;%pPDnw}|4|H|z1#%`*J=HXdi1D?#m_B22t(j17NGAlJ5lu+pU3oa~Cw|B((D z*z6=3+U2;UM8F`3vEiUks=#+wBPX^zrLQjJ!*s7T*tDn+*D7<+{oZu!uhoF5TNk0h zU=AjEmO;$E8R&Aa9Ir1}g$8e{nA}aZgnP#t>)we%QuRjYl&d2nf7(gI{(P$8D2gI7 zpULq3O!gahNtCZB0;|tJlfttJC&GvDR%f;MrB6al;ZfpfU;@KT0=sqYQqWv;nN-xr z;8~v+v}A??ZkJhzR!fQ*ySXc|FX1j@f8q+UFUlZRcQ%o`H3CkM=8|z56L!<#c^ z!+fJv7`HumG4n{dC_~O(>B7gG=>5I9?*1Hl1Ke&7x?w>(| zQftB4t`3R=F4Ex)SK3+>jalm}81J=b>0pHsG_7bM+>cM#k^E!2YMiX#Y zkR^4>{cUeGNd(E`Ik-hm2U+?N%-O#ya3X#Q98?s->d(ShSh$b`eE3aURM^P#-3(+& z1@vXIVdB(Y5_EDOSey+*@tRdAJ}{5^&l829HJPC0AlL&}1#5Hv3wz?oFKX{kQ?7BadM3mP6XM5{mVFkH_%OjLMF(Ibw>o*dOB1DXQP^fB{Puq zo+j2*qFQq?JN&IOJ=x4b#fEIO+S!0Tk1BA`>oy(Wyrk_u(b&H=5Qf-cz_m$c+OyK3 zL^>D8d=ue=&=4JFub?WCq7ZSq22=(X0dJido%wYNuJbfRnr8qt@3uf(%@yMOVk;f! zD@Tu~mcUZk-7YnwfGFJ3#`4NS{I)I`VitKVQog}7C4ki}(fQxrMFx2fkt=(&c+PQXQJTDKfE~>%*>Q%A* z61rIC&ZihliH} zYXhZ0KMFB;Z5~cI6M^Z0NAk zz?(O#pyo&>6o==25k9PiKK}lI7jcH_Jw;%(*hq{FRX>(iW;+y$Q?lrVQZ*>Jhw+gRyBbR@x?&jUBIYakH)6Y5~OqWRrATinkfFm zo2bbjBDAU$PF#9HhjZf5E<6eS^C}@2>)=6=K3eT%m{!?ml;!uE^0zKvC!JhCj?ve|0I1Vp@egu{2I%{TMl3oS?9jd@I={|z#jx0rOE zscwH6TY^%`Yhhu8z|RjC=*gAK$aD9-XzTZ#i2BQ7VQL-@AKA^=$8RI6?^UMo%qKigXLi(P0mr<=0R*HUeVAf` z`O9kImqQY66ZCLPZ3SolU=G-L+2dKkK6&g^1I;g<(K~Z?QsteN==-+_`D+}QhRgGb zU9}3_ON=G0+-&?^;70c!NQcrVf#@3}ieb+lvgfo&khbgb=((o^k0@qh+Q({oNeHQH zb{2L{ibpzA9W>AqO~Y!4|AlPmJ$Qr)$@MV#I+dU@T8kz-Eb-!69k`b(A;?7qEcD4t z^ke%`DJMBNZIpx73#P;HjW)t*aiNcYFGC?qNs{q(AD(%biYlUlUisk;+Id+7Up{4F z!AdEV3yNi%c~qh1>q;C8eZY=!t%1)N1hP62;3_SOj(_u^GeH}LrF0=>nl!Au#+Z9b z9w%eg8;IV^95MqVpq1~7x>am}7aI@j7bcoNaF4^zMRm-=^@DV4<_0XCbeX_HQJfS} z4kN@GTXr}>-8DNDSvd<$S9Foz>Ax74Zf(#!+lbA&0c49tG|Z|oLA7r(WZz#e#_ymi z>aJRZ$2`)|YG5neS5n5-7#kc?=hJ5swzO2ei2ig|#^2Mk;rHzWz`mr9MaBeq#Y=ES zRRa$0UV&)^4{5H6JNu-e8%FWA;OHks4F6OQVUL19bL2NsSfPNq($aXUUlGltO@VuR z86EYRj$X;@q45tNnYDA6I+uLVx%H5AjbFk(we&&l~;Qt0AZLVVtxKpod^dNj})xUY85;oN;N8dwSC zGhP#GmkxAOm_*v2{UV)*9?{L-47gdxf>o&s``8~2{&;YXdMWcEE~F27RE06b_Z>00 zl*lBC6ws%gshD6}g)gMbp&%#?P248vJi$D`>tnhYX*Eo?Wc{YqZ*$=xzmLcahQnIN z2GGBp4cB-1V3tq;db?^tApY@0)eX4`_F>?Me7I!7`G3~};~NG5q|E^~gl4UPq9(cNuJ zQP!J_^X}fI;f79N+Ors|c71>sqBbzUI~XrGOLfQII_cyV z*s0sG zh@&4er&-9qJAjki&FSdmEcj;7iMB7L;V$boW4Z7)^|8*a6)$RtlWP7{bbH#qPlE*vnDFq#AKD!pVetE&tJ|$s6&N{j`pZyb4tQR008O z8em>xGhVzW&~HHtKuN|DB10bFF6f4XzTMCib)T);RRXa>eax{R+n_>781`^dQO@%N zI=@;5q9 zGTruaD_Dy^BW|p5_KF+d+ik6jsNMTcoVhO)x)TNG>cVIC%fe$|Z_SXirKj+ld>5Io zBW9sex`uq)eG=N)bFuy&4{P3Cq7J1OVPe(+TzsPqGIlLv*j0iaUdM)3{3qS<*l#=d zujddJK7^H^3*sw&z?57&G(GGG=N7d>Y2tCr%6mto-GiuB&~frNF$=!)lF|M9Wo*5i z3LXN_S>69Qq-*CuZC4!;djrtHt;4vh0&YNX{zondIHuD3FtNvr&5~Esv4Br*KpBGzcjd5OY~gA`v~Em}k|)hrDL=uvm!Q z!>1tmbP;4zZD>8EfSkb+46OUn_SVZ9SDZLcX55k%c+cYaK*66H>ljn(gL}zSp#&Hi zXu;4Iwd70RH3-d`LZ`=Vfru@4af6y9)5dPau#BthaY3DzxOoFR-Y3CM&kE}NPX+9j z7nvQN76uz%O-6EYCtP172XTg*NyG+*u6TW)@wZWj|Ac?Bm&tqvh4ACBvhO!-3@Jbr zmu%t@F$d2midalmR}jqieI-Fohp~3-0{fO&BBkw;Xb`y@PM?^-ecX5$U38xkku&hv zE{8qoY!~pdE#Yb2NjM$c4OaD6=+i4dsDhXj?CTQ1p!T!adS)z;4f_B*+ipDTH9$Q# zzhgg}Yl#mc{(ziU7M9!(1>?&+v^#JXLKGOdC%6XMu9o)ZT3;9oCm4FQuXRi=H#k@7S z22-uc7rw%`m${V;(7(Mb(MwWMX+$ki=AzxFl0WhS!0d7nkTUA=nw?C#X=Z& z5JrR6W2G_|mwz-txBr$w=pjw|V#*1m$L8YIqYuz0c0c&ijW{Zp0U6q#LLAcUnWt7+ z?C5Y`_M5kfG;BZ#Jzg)v%ZIK}Q7=Ky8`y=9y$`_xy`@C_R5v;dh=R(#9iR{}4XqRz z_+_yJPW}`II|n6dZkdU%Z3NzabR@k}ZiN@K=i!>EG4!MMOX9Wd88ZDp;GKZg8%f`c zD|n*tb6_eY)vw1r>l9ES@G4BQv_gX*H#DhSN9f299bGnD|)Oz>5mGHKhL2tKaQbW+cU=c<$rX@TChGI;V>%TDh)(L zVE%zoNcw&aV;lEiOv7V(!d(>a=Gl>vgtItuHVvc9wDI9BF$;w_Pxw!Eibe3|YWgQb z3>s(FPyr%|8hXv_se3okg^$x{{Em~<_?HzN?5QNYFP><2Tpu4Se2Qk;sd%sQ70C51 zgd5v#!EzB3P&pQe)ATElgucf&f-N<2^f>g!_p*oDqd_XC47BwdXnKiQ$GGSR7%BC| z=&TDkH7O3#v#ya_M1{V`_$KC zu~RO5D-_JBbr!cC0~KgZafJA2bA%sdo{XV`9w|K-_!Gt zr<0Vn4b0q>U1*k|g9H6X>D4sx4iD*vmaTb+>|A#s>XwD#VN3|)|M@+F%w@} zWH72>Vivb0SQe+^vydsjO%#ZL-xbfqRrjx<-}m!asCxuQy5FHqi8P#!Q-#O37k~n{ z8B*tZlgr~N5NK`1K6z;`#+(YLvi1icJ^e5!O9Y|kK^E$1eIto$tMPFLk8BG$gSYkP zTYSFi44b=mk&(zw>TWy*CC8_A9BNsJaRDh{vicNAM1;Y)h$Ob&T3smaX@LLM=i^7` znGi0RQ(rv$Icjz$!v+bd4sDeb611TLZ`-Lu_@pURJu8(tF?1WhFKkB%uT2>1UI8ml zxsh0hSsh`1^Xa<WGM$#uX71 z>KX@`)Frr!9s8v{p!eYni;o*@*=>PqNkTy~ zQ7xMV)}P05kD#Wia^g|A=NQiZ`xD$l1v63}lCUsF#xf z(qJeVB1)4;B4mnaF3t0xd8VS!eD}H)O)82?2@&~;s8EFDJ?|gj>3N)U_t|T$&-b%L zutiKlcm9_;D999pzn4NmFued8_mvU(;D^|n8IAkHlTcDi1P3mR)6lv>csp z%(@P2s!M>E!+a+_{lP8L8ggo8u*7;7f%@ze(&dx`LCa0yO!`cywOK;OoKE1Ehn;YL zeJoBWhruG#eOOnp0Ll(2ax`1n_(_PFIr#`c-N9OpYpxp3OOC(*$62sP^$E0$G4qaT zeHdwf#&HY&N)Oq`K~(xr(slP1v@CW3mtEeh)N{Y6Y|aRm7x1hj%#o-l4|Xm2MHSksMoL}?5!xE+aE|{&U6pe`ZPp&f(dT^?;03* zmBQ4jRQ$}z#QU$VK#9}@xo9o}zk7Yji0UwX;++K+Tl8^d=_Hv}G-L9}g;eBV1l@dd zHO$FLhgIF@(Y<*Ajj8rz^4^h{)pM5&$-P4v_eZSXg?CZWptQQTWC$~Kf~yY{wld#v zpz%7L#I=2zCgohB@Bne<^OeB+x(vED>%oVAzscD}ww%j;6z({;py>NrD$2}~3-@wK zD-FVv@9&eBk2iq>@{#hF$-vtW$m`t0a0FG}}NWj)hNW%aM)x zGf7{fH>?}w$I#&`^yK_jQt9Ud4)rX|H(?m~E9Wp+y@>PkOfNWVbIIZ~Eo|5oigzWP zV0P_#oLR<)4Xg;gx^El7LLiJJTzdvt@6*tE#}?MH^TF`5>J+P8xEwboTtXgaalARz zNY^xeBN_j;u-a4wsfyJp0NAAB_Wid_EW0M0eiM_?C-^owpS#^ZV^~Neie@L2Mk7i4iIO8hau-aXL zW%s!eegA!_9#{1S-uZrTOTr5c$91r(;~}vd_(#SD?s2Z^cCpHopTdG|Kgi#%7x*-*}lxZ4#0C6A1kIih(?ZEf#AI} zP-&~nS#@k3(O5ha>MMT|Md2)v9kaoXq!*mM6Y4C3F(x~D$(wB55r;3rScDYXA~(ek z-yAN$MR*lVBmH4n{x!H<6#xsFxtW^bc_{kn20%RWBJvXK1YY*}-r0?T8=={qY)B92urVzf?FPKaKIC?P9!< zbc-6+%VPDKr8E@En7fE7+q^D}WUgM1)_jU2N_JD)qk#r9na&5_WVgY{yH1H#6zH@Zzl3ieqvROMWNPf zc~qB{1}SF^BK578b$_q~28aD&M6CsH z>~whyB3tLNrH*|hN8X#`u{;%!``HJ7?^>Z0k--@sYH^}*34H4GtM1(S5erV`)00ou z(u)@EbihU*FN;M`gG-ZSYPTtkQ()XFuGwf)af28x1LVI;n7ii^l3p>F-uv4_hWq|i zzmN5RJzsJ#Wo|nJ3#3E(I4|wkl0)t@zqc>%3t5kOD#*FDeC&aaWRm0mf;dbyVMky! zYvP?aG+#Kv@)=X-+#eF6oCi*1)%Kf^D7zkJc09#mmsoHV+d%qXO~NH2kwgz3{K9=sGuSWiwt^}Bc47_|I7iX>l?UUg!;mkgkA1x{fa?bur&dFa;w{W-jwVG%_K>n;sU$gKDJ`{4pd|sqIFU6$ zKdsM(lg3%J=-Na2_Dc^mHLxN3H$RLeEF&KAbGW_S{ZvxsXZ6Wc2Ye)W4?Es1#q9KU z4nC3w+tF#3wn+yFc-@4Csxp#Nl*-gryJ4zUoQ6nkr`Gx_AtPUj6sUC*v3)}9N!~<^ zdNdn9b-u?1tY;{7!Cz18K^vSspa}L~?y#bR9MQ~Z9q>sPW8gJGuDf_M2|n*fJf@zq zd^C#abM6B;xO@)$+&xNNC7VdrYAtlTJ521u%gC$$?qXY~0@Rv5BuYFERC7)pdh*VK z%0<&S-Li&QYW-&U52mn8E>5HKXe(JS^Oj!kp35#_z>RLht5_1maP+>xG-hQS4E4xj zVi*tm81HPbW-?8BpDMvNA{2e>W8hP>Dy_~?hwZ3Mel-hmvurrD$F5Xw;T{!a>_w29 zISoeX0}wFoj2;aK=*3O1a5!@|h}B=_$UeReuj@{dI^j4pa#)F<@BLt1>=lC@roy0q z+yHhMPSKV%_E5HS1~&3`6YmeQc-5wwTKzW{bk_Gmk@GpaLfHii*39IlMJ3TI#?2Vp zyd93$3}Pkk3hI2V3np*t!S?UlXlqX-)sK#+3ylNdTX!yTR~?}3NoO!sWDP2*)WGg< z#$b~cMBDsT;Qwv~KgKCxHDfI~J@%Fi*X)Mx8T*lJ&jio9|EPn)Hazg`0R}v||)I!eDlaKJm;bh_`Vo5EWc(|!f8!#zHn9P){ z#G9$RVaL3gM2ESlYnaScpb;%klRM6L`Ac2-H_d;%2+UAi*iYhXU7$__`O^HC2ck zveiI6D~dUnF5~mdQ$#S|5rjNztDiITAzSIIkhMPwCQdzq>5211cGg^)_IVIfD+f3o z$GIGPPho5qcL#yUQWkw91II-oVa44RNGo0d-A~FO%0~f{7IZ`Tw@#?!2ICEe;kDa1 z2TrzeU=gR0h&8Ok%H_Hcbg_dvPB!32uQgzQr-qsDchTpM_S14lIhuaK3HS59<9Jv- z#|tBa(5sO~_g?!!Ebk?A-nzPzYFllzzHQGq7X)ZdF-&gNK33Az;UA z8W&Omkx@$69cc@9BsPKNdOBGlj!a(K3A5K7rfcowp#6;?6rMi-4jQXrV(=oTT4jWU zT4Yi4j6s&Q&vhI*c#pndW?B9750Pl!4Xp9>rzGgaOvu!I44>}`VSv;;P6gv(>0D4u zdL85N^rdWwA~kUCWe)6#sX=4^Wu(?#8W$Q0;Khv-v|zzF#IE$AVatnITO*Uu?BEF; z{WF3)>=RgfMkR3dj^p6(<;Kxi(Li#$g7IUWEQ(pD;efLbpdg^9sePuO{K21XQC5l-bE=o~J^_jjJK#`nD>`S)LN?02RB zx9AuywfaR?*Zab$ZQ~@lEgY+!Ow)Ij-L$PGfZmYV3jMSlN9_FZ)Wr`dW8#G${ww29 zzXD9n;|IaTSzz*RGlq;lpsjo{^vX$w?-)4C${jd`XKr1?O-ygGQtbl$9;^xN@juhugjnxWyx}v8z=Txy>O_Kan=~>!0uzb z=(TGTD{++-=xcqfcH6sxi0E5@*R4do$hBHvHPuXG2gI;+MZD=qkt9~e7Zc)oECAIv8p8c8oAK@pOH|GtB7^<$a9nIIw8UqD zvFkR*Tlhw=)}n)xGH3zg9-@%p&76IU7=CbRGMJubaG z{mInsn>n^^4gvEAoiuN)7D|d9A*|BXs2azQ_xCWomxd9htgj@JhnB*mwLeWL3BxS{ zF`&NuGMewYOdZR`pm|d&&^TutOMS$8Q~8)IKiW(qTaCc}b00Nd7!91?X6PrE2e!{H zp%>qMy>%K=&{Y*j$R-cGz_JALcB2DE_?BjUo%>d!j3%HKSBo^7PhsbdOGP_m><=1$D z4gVIjdYuT9|1N;Aj32rEla1vDGvWD6Z?x^OgytZBSjGR6%HNZOg33TRyj2Qg*Apg( z6~c)(PP4uhnp$t;SLOmwvzm^v=q~!1ahUAVv&Q9h>!H#o z6Ibx8L4yx>NY>$TT3fvZj9a&Wnov5*x0W#8%H#B<+YdUWa}{>Ix4_;T`$5%hCh8mt zBTmBez-wSBtElK02>SAX+nbHdJC(%r)3-yq+zRYi7)ureU&a2}YbmA#T_L< z@LrjD{#=3qJiMs<`XJfE(#ioFW|oqVC2#maY5EC$Z<&tzX&oG%4numn`x~8A#$)*goAamkn=vlc^_8Q^pw z!+XjT;E^w>_*ubZK=;})ee(a;bU*Zr*G$pQrfd{=6MteFQV=&%zepIIz!+0Iw^X>6>jcJ23B+`^iBwZ6=aGD1?81A}u zCqG=SUjgCmWi-Q?d5@VH2~U2Gp1r6fjLF<&4YqBC=3su`a%7J zN@{iO6&2a>ng~7Li_K|abjac;z4;{@;}$Z{HBNwNWHWh^o{!WpaXlUnzfI+zhm#@6 zLqxNj6Muo6`_pm~!YEA;&? z;Frl@yet>t+=-)@%=CBPyfy`$C-y|9C6pu|Uw|U>n9Rn;GGfCs4^lM3VaQDYMXn~n z?X|wpF}i^o|5^>Hj#lVZ5vZqq{~E4iI{x*kD(HU04a{@T&=T1b*{;`6EXEIv zKgB?-Y9Ja!2|!Mq0~~8vi1V^FF=ba5c`0U#V(bWp%{Ym>h5e~la9uUu)+}gXYD(KD z;`mu*a<#1?IU}jPb>*N)9h5n^c zSiE>W%2g_XIa6cs)X8z&E{EVM|7j|3mJV+%($W3QN)UX&a7|AGSmWh~Q0HSfbbL)B zrKj@A(eug>*p!65X8Q0u?m8rv?59fKRlzAP4u^cR37;j0@(k{%_IfQ{U3Afoamt=0 zKdv!zw72n8_*4w_ssO6Dk&kK+JP{CNtCtoMyaN$>G7ML0Yp9coXfBaL7*Kse@FHB?{kBqm#(VGyRdg5i* z!_rx)OH_{+)3$;b$`U?-N84{wV#}p37bQ`p#R|}SCmG$gE@oy(5hyQpo7D_qBxp+j z>@yCf)Z7F$?2@R4s~}T1OtZvokHN;YS!kVl5tq2=;Tl&SXrG9{9S_8zRN){z)A>OU zgfbpVsQ`LG_YE16>8I+~t}wo)Kro(XNkW4giO{TYH2!E0W5%yoX>S={R!Tc@AFL<& zoBdHndIzkq<;8VhDe-!n1}k^v<2NSjt=S=sz4a+*bMzM#2@B^$ZaxOjDzBisY);PV3_Lot049a%sj;jP$j3kA2#N^N%PCAwUcwYhgDg<~03Sz?;n<{x z*5aK(DO6t^3bhM5^xTfegZr=wc!|8z%lqtzT}de*^F9k~r*v6r#Vhc`fCgUuybS5I z6Wmu}+z#(OAY>k^Ji~ucN%-X1V=`pTY6^Q(Iio}QTs5(#sO7sD@?e0Rj% zLeYp<0A;?51HaKe9J^xzWAbs(uehJ2zE8$MZ4Y=Yhpe)Se0<;LiwbA-AiGEmyngSc z%NRzgZ$lmZr5S|d305$q@`khvWP;ve=FU8y3`dLJy-%FsN z`g!Q}Jq)WpEy1dTqx4gYF!HXr2Dc9%Bmd4Ztf=2@_$={`*c6Eo%i>-VXt0w^?|wtK zm8y~TS_!ag_iAkDJ%{&^aeN96k#Lg~G#iOVzusm#xgrbmB5lbvu}0=y=^@@e36N?=eXdb|N_viLm#B5gbo5 zfxi>oD5@q*cz9xA<`?8hc-+FX$9?ht`FXz|HRxC|MK{EUvs!xhb%nO!6~ECxEx27sJ5!|N6&v9@Kd z!PzeBsEA=4W|ViMS09Ur4-Zi>hNF_5A&DlPsZbV_1xpXefjYw&&A(s+`)uT)r97V% zeItZ9KP)NzoP)K*4?V{gvlf-F!<2LTuyvmkUVT+ZBVVUbPsY8a{d@`N_MKroYN51b zr3Bs75kPF`V$!f~_FGtcKY;B1r;fJD_h`7mS@`5zg6q3=p|PnI^VhGyTR%2~ zr9%_VmRSl#^9^9WU=95h#XOg-|47#^VQ#Q-Io)&lHu1TA1lIaF)3>LsnctKdv`(7o zb?Vl@mH-u0`)7g{tAa67NS>73^as}dXc{*-8{D&hlSSW+Q0w0SJ(*mIjdWjF6?0gb@Vix=9Wx!7-6JR7& zAIDvCf!{BewJ5m{%GWPN$A=nZ-ynyazZwo7oFh5k2P}y!&jZ?WPz9>4T9JU@HawTp zL=Vl+rShM`VQo(!?yB{~%y@u6!BWtuj>Z>uBB+MJbkB@R=smiUlRK4zeH&6quG0*5 z_pSTLPyMlL_aoMmcSf|(H?P|N=nImess!+*h?%(_ zKi7!eH{DnZ<}i7-l^K{~kceg%<>+g&j21kpG zyqi>USRZ!x%d+)v48h@{Oc=Ekri;Ce7>9E%L^lLtcK=#5(l)~WF%Dc#xD1P15b4JTP@!ImzrP-VAL6IsS97MJeR-j$+K0sE6K2bL?6X4>G4t zWBRF&7#?fQZ{df@Qcp{IHEDUDki9I}$=rr1>Y2Yw*xb`No3}n?{K2bTlndya6St##Ce|8+6=v z!OyrF7*9x~Hs5Aq>$Wv4{q7fJhHo_1o#w%d319JHXA^D|Q$o3)E_gU%2;$A}ah$tz zz}P*MG&|aW`}Evu{!5J8H&~ne=s7_64DulO$$QwN{snV_L?JN#36{xiMN8iX66&c- zno_NId06a`Q*8G2EH;dOdu=&zir28DjOgSg{q7B)#xugBCZ9w}$hx`6#$#B~+(wc#FRR>_IPI z6&C)K#wR{rI(EfcT(v-gRU1~b*L0pGkF?5wZ&5u=CNN&;6jzCfhSLA3Dxy5QXA6^1@&T&fYRf1G?s-@USCY{c#+&{pG>h{Kt4`QyPAm$9NYD zwAh^vCm0WdJ?yO1;^wQ&VRs%l193-6kjpoRt-hrlRxX~!w(d9qe*)aG*nKv>voF9s zyR~6dZxv~Et0xg6&XzRfz3kQAEGj&w>`SwKyY>sW}#IVVowl zI4`piPH&N_7C)&DV1e7#B2Ay z!rK=T;8!;W*AG)j9u;7B=QV?x#xab2j&$2`eKb6NpK5Uamo{AmtLKVja6Hb{Y@Ln z!f)fDt*^nkH3M%awn6;SN_4SEV%P#dto-^1t^X*n?}$jV#gb;Q4fA=~vbA_j zH4U-e?kZ}n)@F4dn2FY7LqxiS7w_dQp)O-@>Fkan+W8hi+oAx&^nSvP14ZEXK9qVH z#u1nQuAwTw94G%s1YQ47iA$D_;v(5H$g>K^-L_Bh%Yqy@&bN(muVg@eat%zon3B+z zW^Dbz@WSJ9@L2mPtT~`XUpFrR&6o?A`Thf3zQRk~@2W$N<#ve45$C!J?V!>bY+#wG zW8(@w?hVrsh-VE`!D~Y7Hii4hSCWd$wa;@BLUT1r!hqlz(|Fe&HQ}Isd@aJ@lR;NtJE`iGx!{eR0^=QUY#I(F2j(R`5dkb?C1J1 z@5Ij10UUW5j1hCgQO)X_ZmQ)cj8wRXKf|V&oX`sDzG61J!sva6UL=)_rn6rP+x4;%5Ym(J7W+SR^?y?>Wu@njkH`6=oR zZ?J(DGQn|wff%Bsz<#hW5dO-U!)k@OT!qF}tWPHc@JeGbSbRQ176cSxZj~2!c|0Y& z-TtVXeVp5Lk(b>k(SaO`Ug!>bR=uph4(8ji$s8^}_j3}%Q4M#4N#|Dl^~wZgrT4(E zyQ-{o{d%0z`+~l{Ob%Fs%etSu4kGUPg87p37?yqjTYrB-HHBX&qu9%-uKPvU%a@XO zdKy?%C&NwKKZb)FdNHg!9rRDuvu<(Yi2nm?uKiVIkof70cY1E3`#2B#=E@H+d_kYB zQ&I(Ai%q#vlM7J$b}YGYx(eIO@8Z9jBe=057a!d+pr@Ygf`%KJ;D2osj0MQBw_3*m zkJ(DL@Tp;l&0}%BJ)P-`FXPob&l73v-MiR&!43-&FJc?}8E7_2aCuJr#3*xd_Uy-+ zD0EAS>bLp8XSIuT3FD&GyJyE0{`e6ZFWq1*a+Zh9nja{Si~%g`%K?eI(o8R?9#$%e zqx6tC_xE*C&|8tnIhvS>q0q(8rBF-NxF9?#Jz%tF{ zpi(NvjvVG;uS^aAz5{$**QIl~&-4Ppy~-MMuc|}gX;VxNb0VKbuTlQ=Hh8#i1YSS< zL>u0jv&T-pqig@}gsE){$*7(pw_flGR_REhUgI|^8zn)K-0M)*QV4CY^h3tRZall% zg{yQ=ggw={4uVdrke7aZ>}^kjtDBwXaVKi_b440j$Tyu9ILhj!YUi8q$;nD+RF!0_ zN6X<1d3_i!l*iKzI$TSmaCq?6gQLAi04hJ9#90ecKyu+$=Fa?r*q1NF7V+!!wbpyQ z=;^>(V0w(xYAnd|luHJM-UOOi&#=c+6-j`&7$Ae<=eRdCsyt~PX4BElo(z^jd79JyI zru?wceHNufH&HbG4_WIArKd%2-9H61D_>&FyEMn3_j7nKogy zasYX8Y7t%BEzO-9eS}jd7C_D~X@iUR=fUNri@B`KaI}ay03ZULc1)@&NzAP}o?)g_?Q+R5WU!nI2Nyn!&>u{!@hYa-snz z-zsn$751KAR=s7js=R{RMz7Rx*z{aUT%+uVTJYFRB0J5$0Ud~&G?&&R_3AB z?0Ae_W<)=}@?o;7v8;0?dN4ap0BVknvJ~(uIb5DelI9+TJ+HcHi9-&~3YSg@x**=Rh@;3&6?bJ<(I6+ zyv?A^+$#h&9>x-_9B{s+2ZQ!LEbo@CYKvRac=PdU?y9S?@UyWVPaURIHk-xWCh`ql zmF~tR>+eGMu?!3^p3Rj(Cv;(DKz8h2`ndc7EZrChrmuskp6CKNH|>BsBrm`h?seuK z&fKx?6hYOl@8B)vO~OCEqM!9-*;I4{G-cu;?ciQ+Q+^z@4eSC7zF1D!vIG*essbu9 zB8c1gI`-q{r*LNTEl5fCfqxc`pCWZIhssYn4^ z@HpeEKZHrOAK>tgQC3@)Iu{FgFz2NswD(1myn$n6?7JYnelP;1EoP&;Y#$B}i^75d z39jy{Ww6qv9!Cli;MhSwsvXyfefo*`G`*1e?)uJ3dHIF-Cntfx;T7C<1OE8ES(z(7 z9!a+Z9|1l$vnrDfX`D&FdF;zNT_8Q54XS^(gI|Om@ThsSIu|bFuCeR_wfkB`Q|UTh z;*EfPU$;Z!ti{;aJd6GBf(E{MxrKe}WCb{dd*ZvE5@gNOW~KJjAEiobL+r)p+fAx2j>#|g;()+Rx*|`HT;}i)wsuBmD_rc@w#ipAyy*c*-=M4o<{CF|$X_CH)W48hQn*{H=iJ_#7JM@{z2x%mRl>d%EfE1ER6? zD6Hx0W}R=f)N5L$KxC~}!Sc{W>;R`IOxbt=)+}jZDXY%mCP&-hM`d1a-nu55mTd(+ zddKnB2{*iTOam4xcT&k956F^}h8dUM)5HApxkn!E#JpisP`D(|wLEhGC4axAcPg)t zi0iy;ryO4NWn~aKt{aZ7U&!$Ox^!!sC^3Diitz(sctk~!76=Qn<3Fu|(B?e&=rSA3 zu4jSZ>vkAT2*B8@J=H38<;48DKV*C7!e~JgH4SEViu9~SET1?tM>@saKxw|-zix{J?R!`@xMR~ z6>n-fp3UTT8PD1mc|60X2plzawD8KpNl|8Zmg-I%c9_Y2-@s4ZHwEb(n9IiE6D@wv;NFO-1MRi07#UK_^bcwgj0v9K z^%J%|@8*QM*+KN;nc!|}!c~0X2mAGm(CPUKItDc`npeilW;_@7ht|OCXIDVGdK>;Rc@DR4 zYhXhV!zG@}Cqlat@$Z_!s!jtbgsIE8Z|V!jxuk(!EJdhvP>Sv(@SG%~849%yi5eSfw9eY`bk#_}f4ErD71vHmitO!0xz&~Om; z7sZ<5C&2Gm#p;w=$e!3F0MZGapTbTftoI4*#|Em;`3kiz== zt4TP2B*VgG5t`IMoxV;%z}zCvXy{QC6@AB9!T3dc3r^5iIyjujA zuHcHzm|7vWT~I(O?HCo;Q(haETjCoA0TJ3anZ|)>(Qr3bMx1Bu6)VR+dnpj^;B)1IPO&Kac5e-tJbnlJrLJM6qCKZlrH@p!I>5%2{$$sM z7)*V}4|9)pgS2laL>$zIVX=Kg!B#%@2jOT~ zUGRoJ`60^=VQO|!-56A`c0gg*yKv6oH@#yl$xZs>02#MVz~sS6vSD5gnml+1`Um9E z^2uMi+hZ2HLRx@rckm+~S9rqusI15x9H@mKRfVXsOAQrPzr?)pWLlZA8gCsHC%an8 zt3-rfvkE`IWOf-X<+`M+>sD|0iHe&Z(4GYd;lKF6tmiuUMiZg^SQ@PgAHuoQ6C`wj z7WUNGL5XQ8zHVJXbd%!AtzCW4pgn^vWVRC6n?pbztT^+Yd?Fq~y6}+UJ$$e9kYs`9 zC>b)7>^z+b`}s=on^`FEnDD@r@@Y<>aT&ei>{ERL!gTW-K9Q9Vm1)W53ba;zh;P#V z(S1z+_58{Luv+wtif#9YFXxIN>Fz4L85)Cc&mCrbg|%c)rVgIWJPV5-`{OR140`y2 zG}bNhf@QwGI6X}nA7(b^Ng@~HyON1?)>e=czJg~n@91qBw*h!EMq*hpG(_SmzR;Zl z-SH3L0cljV#u25r&SHy5AyMv><_sQV&e64Fq`&zz4D8XvI@@35ZT18;SP+W4WIGt< zI1`jS1L(lOHT)`53m+daUay=?S`fXIW9LM;aT!11^_ms9CczD6S$MI`7Hq(smh*AX z=SMg{<2m@T4}$0G&FDQihuakJ9TskV!qUksBbh|VCPumK&jE@tEUq4}cSvO!d5BI71dCdK7f`*|DV7?p3YQ=@f``6GuZJ81U;T5!HKi~Fgl?D$LpEhTI=L+MVvFuxNky624CwP z7n#Iu%)XEJck^-RJEGE&AFRJ4{b(KahOUl00AmWFoQ{jTS!=HbfpGB_cxN*SbDnOZ z-(I)jukV(4<=AEVwD}lJW!whCHR7Oayb~3jcHqr7_sQ~69@IZwOg3(6!}T4DDIN}F zc2~@&GY$8_w#Qkx^LYS&1Fb+T@@H)tLggG_cmQRr(X1}FI;$c5q9nf-Cj zAycqLvl%}ycMJ2=S7?WC5K6p=hVh2eG+tB~J@-qa@}f+vWcu~lfpze5gBl8-Sc@(~ z*Rb@+4%}j6N7u234+kF% zsmW)!Y!S?UQ;8|R3rX+zb7J+z0sEyxK|INj6K^8LdAMXgnZ2EHaK)^mS2WiWv)~qt zEV)62W^gz=|3;#QjW1~xog_pj%q4@ou?XQj{#ylNlYMk{=ti*dzY7OD zBd{|ih-m&|GH}oSq5RJQR>416@=QRG6YcDX=b2t2H%ktGE)lKncTU46OLoGobx|~@ zn+;;!_vkJWW~Xi7H5w7fe2Z7ir!vI@@abp)xTt+5&&T~)4h3GU%iD9YrScA1dF|r3 zFAsxBGjp=H)&tIM=!E&Fz40u=Yq~6p2J-?+wunZMFU+1Jm3NsqaE-Z39JYmmb3#z^ z+zWV%34S|}4jE&UBvdd9KQeuuF}@`*{sw8id=9A>(1UHKdRWCqCUg&pWOAQQI0qup#2Y4|#s^x0Y^uOQCj^vaomdwmV>~z|X zUZEfKB5ia@M|~oZX<7p3jB;s^*?#Q%6AV)$TY&a>g0-4EPOlCD4MkH}=hRI)e=!bq zwI+J}?`GJtUxsnhA0b}+^N3`<4{npbM!XpQ#ck*YeHphIc<#T}O?;&V=JWSMqgyDx zb>D-Zulf_|S1xEVBn!1CR&pHG6+r&%Dk|{&G(K@kM4qlsEDZ-A*3|rP6ntApJU%da zrOT1{#+&*3$m|v|P6G9wZ?xXn9$E(EaUzrlg*6}18*Zua=7=jtJ%oAZe=s|sbzV~u z$8{t)#|U>%pMu!MQ^X*X2O5_xz%@r1cj$J1Jl+%oQ46o*hC3m6^rIiO-DyWY*qLGO zp`~~wBpK3QDx!J*aq{`NE(*>FrzsgoMELiU_Fa7V^KmXF%-2D7%XZYbas-PQuV)ZS z15ekUs>o3pP`b|e&}PL$17vh^2YNN$IDQNI6C!H+wtpk_Z!AX=;i`mKf+lyA<2wS-4%G6zE;}?&Lwu{Nhp%t zL(PK>V6tuvoHO%79=l=^I2DaNv0quu=OZDAZ4O1N4dLZYX{hwoBx8m`be}SpaYpK} zO!d-nOZ_sKEv5o3wkuJ6>G9Tns*T=BfJ#Sf!YjUx+ zzMYjg?nP_!oah_23*$!bAU`wrVNY`=@@?^k-UR_9PB#SkX6;7(tN{8YT@?A{15sHk z3bt=%VW`*}dUWwJ@>W3&E^9EmOdJk#bdK!DNX0Qy={TRLKCY!>*L-1{6tkC>?+)R% z%0OF}7BpMmfZhZxdU}lr`ZVd|(MvoK*>sd{yZntP=k?Q^L#IGyQ9Tiky+^y|?E<^Ou0kI?vq*dS8Ad**22+xT%j*S#Jrczt(};#TBd;ptFppV#oTr2k~Jk%?0GcIQ~W@JuO-p2irvWP zeV^DKOM>&y1WEgxbab!#OUu6PhFOJqSfH9txqipcAxxYadX1B^+IO5~9?Bn&S>=h+6C}-g-dhBWmDRzHCpPf}?GDUZRMo}{iModAA1?-G1(H_)h zpN8A3uak|xyKt!Q1ceWdSUZ>lRyG-6ytN7qmbX)ltu5egYz`lioM88dJ@B%|7mSas z!F5M`NLVYtZ+^rBLP}s3;*1{hIpj7^1}!v?ot%p{>2lLrzpVORToI9A(w9d`kZplB$EwyJ8>s_Uw1wt$XItC#~{M8{yz@j zzC$H6N$0j1e?bf0vi%C%!U{nl{|9+{E*RFuF2>LWb-wUeg!B9|pAFe)`2ctiiHiD-7a;TN;r0FlKf$zm@ zPF=S+w^l|C?9RqxdSE54auUPaYvz$cM}4>^7(u%P16aOgW|J~xGqbGd%~m` zT^$6telPhTlKnnqeBq*&(R7Z^sqf@=Ln-W1WoLG^n>Y>koZ#-^cJh&rk4b6QfYb%u zL@(tiBlvEDE5zk8?S*jgzpLLTK+ zsX$icPRP{mBeJ1k^#}SklR!&<@M4)H!AWCOr=*$_eaMR(Pe`G`5yNJaLphivB?2p_ z&SrX}Gs*k%B(^)gnPv^EaCWkLxG5vUq_nLAmJM~0|9p0Gf{YiFgqzFZqLD7gO87K$ z(=8db9xsFSY=51j#gDoRbs-dH%D35(7G&mJlRgqyCpF0 zftyL^X&+c=a+;~wW&jJ5obb=H?Res{2GQNZ`kxHlIjOrfF#BYe`ASU}Cg^hw2mQ)O ztA;-rxyW|RtO8-=i3_|F4up8_Lq=oYT#_VYkN5A$z@eqjsiNTzI$v-TtUWj#ZydQo zZ@*z(492RlKF2qFb2gxfQaQ1T` zXxH-Lo)-t%_u@sYI$;GS#RU*_tRF)XOzAs|G}tgJ0?Vf^L>2i;Qq$^;-oIk-CbmEk z*MLZy7~tjY?{Kq382j$`A}y-Ptgr3@S~c9D+aw1;SaJ_I2dsiOgRCF=zyr*bQpD$L zJn+^9Q79qROxl?UP>l(|kopol+N%U}yj>u~y`73~-A43&PH@Z&C3)s~?3}S92zV@) zc;&Vwm^-->-ZO9L-Z`vG!ErzN;x~#DuN|S9&55%;e=;r-OO%98FkZX%yed0Gr-h0cSYpEzu-umh!&*Z)O#V}#&Ha1Ss_Z>I!Vh>KYW%llL(Kz zB9%`?A@fu&9u9a>FMT@*tzJiCWPLWKRMxjFO=-v>C!*m2%c!RlpN(j4{mjB&z;rNzy-UB0my;p@a^X%(3N% zP4WTkdn>*El9nttft@KUPWUhfk0*n)*b>~gYXl~H7C~%}Bw0FXN`zlY(+7#)Fxu!W zk+1uL`xX`9f47#?WjFWHZxy#-(5HuWNN19sF)p2$6NR^2g-LUQELKI0)>qiEzONa> zbV{i(I%LL?$0xj*?~V=BV`3T_2WBx!Yo$0r8TFtan?uv;HMk+42Z2{%5AAH9HDN&y z;vNNZf}Lf4f2#;{4o`)dVT#QMZ(#~Od#%w}DsxhNf`%56251)ZH?F#Bl+ z&0pw?Kh&h~##;9Lu~Hdqn+@tKHUtyaF~n)U`Wb7yEb#oiA@jtkUCgL$9cR=smGRaN zN3mm9sP55Vj;_}qY>82Zy)SPN4ZC&l;A9Z%ROaVt`WAwdO%lt+|Bqa^d;-#e*66(a z6`i@J8oG+vr0w}9=rgMXQeT9_zT$RL?=}Wz4JEknWE$&UzCqNsatMIGnSE_J33T_t zE1zCq)TWc1HJ#qzZR*K(6Q6tO9>=p& z(Y!^OrZ-K+=<;e@y|)>cbU1?1AuY1eawe2bM8VRoIPy=IViaEp8jb&e=RSp`vr&Zm zB4jOiwC&;48vG$&db)_M(^NQmtDN}myMvmyYB*=kyTIX1bK&7QA5Z=2R4((q2fo^w zaHlq$f$~`;sKpiGtr1m$ypdJJh~?-$IhqFcYWK+zQA#BIZ{x8I>f8{GyL5HBZE}5Z6W)bb;+=NK6Cj4H+!C4Q&h@GApb`Ca^ z(C;zmzfznRG{JJSV*_CG8-CtkpC>M;m!S{xRiU!}Io2POh9~D!DBYsWo7E6pZ<6~S zuKj+9_A)%Ixs?ch7uVC9WyR>8DGM|9xx$&7{4|8k-*_LjA)gX|;-9P_=+gCIoOXY~ zo8hC(?ELAxL!)z`1q`3^8R`jt2)9LM%Gqp;U!Hh#G99V2f|k{goim{x@= z5T%<(#|Kq$L3Saoy?ly&UQUD2Ke)xROLC8wI0;N77xc<*kaJ?N#YAnpKdzTPz)3YY<)eVQioyZv#rKa40kw!9_=K|jnyx^^xD-27of}pbfH0#SOUfM)Iyg#YNm3NRw-vGk)`*O{_ zZ#xnv;|3TO4JQGr(!8V6UGQ;RB<*{z$BXG@@9KTF_~B#^6MHrUYl=>yx2+ev*AnH) zRnFwfc!oic-*F7~G~n4a{swg1MQB?hb}ge&zwQGpJUt2>PAoId?JPA8Hs|$4xTD4D zVVsjGh!NaTjPH@h1(HK#W6e5PIyg#JEs;mX9SYo#fF)4fC(B)3rHFY6S=3>xKOGz! zq0G@Lm>t9=8_p^5H2lq|dBtmxdYp~{20yWS-%@yf8Au;@F?8N&A`4RWfPdc*-dlML zU&OaU`-;o3<-s>%CvAnxRYKsipcJ<@U9tZ1m#vVPSxjEYF)`iv%Q*;0-6(at$$IfdKNDd6IQo%c#Iw`*cVh}&PCwBI+X&ovSc-aXy+ZWo37~kqAurTXgj;gxG%-E?ALgW} z(^~UE)`hhOvaDv9Q{8U3H>DdzGa4ARa6R}fQG^k@`^hld1zS^j6IMk9n*aWlNe@pH zfyj16-X~>UZq3Xw=0-^uk=2pmrCzM2Ce43g;=m$wbra=k->iY>C4q4H)F1f1Sqoj= zb8y^-M>?2MS}e1bofnICh^s}67NAP-tK-W^xo75qn1NJ64LN!mJIztq`7u2EFUIUglibK6Ruo*hfp+x?ml&!O(1JG!(3sJDNW@4f@K;P;liGB zaFUS1m=2b+8t9Hjd_|y?xP-f}HW1vNMPX@Q2$A&JN0eI2pnSL!(#Rw5A>J`&_MZ>5}G-Z=5ow1jt9j`l-M;Mx@j?~CT}bW6jC$5IX6{Y)Qb`;#lA zp(YnITWo0aMGM}oBnd9J`8pgbxrC(-s=V^g7ofHEKI6D=9?woVij>dSE=*_LKg+qYZ8J%{ zSSdV7XP?&=eH{CBpR9>?B;M?ExFETRIuUu^^}j`UNK=R`?OcQ2rfYe>+%H4Abp^z( zw7?(1tq}fP4rGo8HbhMoD2nb3Lx!=22P!J9B-dK zfG)pXae>@=xD;c71xo9{M(q(!3VtD9J))^m^dD^J8eoaWWt7}DgUeS?h2p%cP|Kb# zGPdpm7l&}@KAni3j3(x~Zo*Y6E%>Z*j5G-g!tHc5-q+MqctqkFO*|sRHCQoWel(qd z0)BR%yfOwfuclBpM`SayD?oH|IXyIF2af&I;L*y@7-BsGyXW!IR4Yr=k>KNPDS8O) zarrEtS^i5@<~2Ge;@gsBcm%b?-lP)`JiSAY?$m~mfi!%RQw6Fg zL$UK$3o&%Ri7|mCezHgf`Rv{#0X~{SQ=VjD+e*ItI;#OsVbce7y)hoUA7SbvJ;Q>tc3rV;MA0=rYqz^T;1B ztJ*iMKXBP|_B~Np#oo6!As0G1m(^hK!_5saLEhRgxWpGJ~K8CjaqSwW?gLdyt znEBx}-1j?=2NhMgh4U^l%#UXCdlqMiMsEb_T`H%BVp*8iG=N*DykWLoy9LE_pX2@Y zwe*#17S!$xCBpOna9$ul;L2BK^sgD+dbS*mvU#Ye_!zG5_NUjC1~7b*b*C=V0B>1_ z6R21YJstjFvgj0{>{FN=hB|P23)Q5pQOj|HQ6OM%IWqn1c!Go z$O)6*4EoH0kH-Bt{O%su%GN@eN*(EVpg?)?*Me5KWtnZD!UJdb89k|o`9YnTVfGJ;OdGcSZ$g_H9y7N>Jm47A-6@5j_wOA=` zaJ@CWW#2&ylqGp<%2#7C|4ed&%?`aAI*&=)FOw5VArQrhr}ocYfY|ge!~hn718 zKev+2Egq#`v=)N(`Q`Nft{o6*n}#3RoqufI4u6R9XDNPx#@$+&@=W2q8yvy z+_U}Y_#*-Zqg3%ky%x{-`?%RTsDT|n+@%&XJBK&3xhgZ!Lk5d0*L zdIK;NL*cXac=xF~w^>t$+-@2{yMgB*{f6!B{41qR z`S)Py-#oOxufVmkH9^}B4S0D|7CpuK$Zly>&{~j%A!n-K$EuqsF0PI5BR=D$)j4o6 zt`9DrmSdTjIy~`cCp^>s8RW-eh)qW!+}X$G7smT(2h_ujt3{Z&Hx@tp$?)zpe+QS} zMR07{OW?+YarC}g;eU^Av%lXAee9)3<5L4@?F~l1h9SMKHfJ5OJj+u9%V!(}bUp z;Cg4)aUn|ii=-gCCyrAxC7jByt;0JuD5buaPiV_5Ed{6U$u@rr?jp{t4H7ty{{)c{ox=7^?hsieU$B}sNtWu)1RZZp{CJ(g z75~10x_c}@$6Q`|V-(iAKI5EOTLmVG_aJMkHQf0cf}E9)!LmLId^3c2bcGLmd9c)c zZo()vb&=%fwaiEv#0DukIeaFDN$1dA@C7tOnp^39I-xccT9Wqy4M%kQwGQQff8b3Z9#;*&bNR?s^2K>GSv)BLRJijQy`zaa< z-}L{%apzUwo+b_I8=gblHU*Zy?TcYsGq9twr{3SPlGCp+mAJIt<6LOW&wOiU4f6rAHeS0!jN~$ zfGnL|MZDew;luwTNwwoqco-spCP@}V=e;Xvh~DB1ZhA&Yj3vv?kU~jSVczfqb*}Tr z!zj=0_`C#{;^~10gu6Y8QSUT^_}-1gdgg5WlXn^aT^GV%62&+-KZkzZN?`tf!|*cX zFnM+71A<;;4K!m>e<>WP zOvjA6Cq$}M6(s*o;U%qqMh(1WxKhVs;Ewuv&We|A5V}~TKCMn4qgeONfA!Lw-Q^1K zM2>Z~RPjT@>l5Tko}9^BEmdAoI_u)K(Bs903iCc2%QHv+iej?TTPonBg-0{|A(s8T zEOfq-Fat|iUzmhylJRK#zKF>$$wu{-4PFGN~y$tUDKU)e4 z6^EJbefKH%uOdhkhr*K-6CC|hL-&^XfZb;W?#?tmuBY`|OlK;|;O^P@X4^03&7WR+ zYtcS15{{rZ6stJduO7p^l6ZI+pu{VfwE&-PE`g7?Y{|&gRbaM129D|Q!RGGmw5GxV z70NW=^k5JkN@&Nrg-_v)ObQMcgcEshGmu=tqlSx?VG!$T$upMZojJFWnH~1nJUwU+ zx=rE7wj^EB(jvhdK0(0GHVOti=HSCkmBeS?2Ik0)F^J4)gR2q-^va%a^6P;z*-@QM z)NO@X-_Qdb)7=NoLBsT2z$SuIZc{b$Yp7(W2=`SBvAcZ{n(wOw(N($lK_QVTKl_5Q z5}b!yJ8R55`c9Jfv3Yp?Mk%^Ja7Nc3TPbfM72O0((a9?jV;@Lx7cJU~i}p_C^0uAF zUYREBnWG672A^4WzaUq6dL%eixWJX6k5uY>5UMD!o#ibY63zbH^z-{D^^oPYSHwcg z@m6v-Q51?7M}pLBEo>B@WSqUaA?{ZS-L*6bSFrn=(RqRxn_3KqM#f3_kUZ{ZcYxdT z8_@R0J~Z2(%A~qRBR9VRz+){AeDx3ZZ=53T3;Topqb4?Ad!KS&m$Ak1g)!zUZ}V~p5W zy5^4}oKcU0cG(h=FYUs*2+xye?7d6KSKCy}ECTZ__TfujEuNLzj4j{q!LyBgyw`<6 zxV>(Ko)=$F8?%Jr#+~yVr2=JGe3YNJ^vfxD6CzKx)wNOWrfxW@+e`$tRk^~q5>e;x zJSY;Gi8|67P=CjM*m;2x{!=TNKT5aBrtUygTqsJzI~S10PYUqiN`_oxnOlwN@i_ie zk>{Y51FounVP#_#+TUn_64(FeY?fL5yMs9os{UbbwgNz2GZCG3pX7(Ghm5+XbjqqqN=-}XuHB(1 zw{8Iyn#bj~99lq(OIY?f+K~JG)fj$U58Miag%jl+;hZ8B^h@rG3c>9!Ma6=ew-FJWy=O4!W%sHe{^CX@1JsNC2--T;; zkqBN?=4QLgVd$SOj67}yiLq)Z_irZmq{31tVOj8+oM*V`Z!mFsrZqD??IC)>A% zYI^n3@=+}uR@+NXtZl`?ymc_^?mwEm^ACx>MRJ^)H z=K9ORwF6PO>7gQ5IeI<@o=u|~p^YH!-bWwr-44H|h{F)CmAu+*!HoYb=6tixz;|P8 zr@^9>S>zhXSl&wk>qGSrpjX3YzE;DWn_*Z{a}G_DLLqwp0%R8OaNy-@tT8!5V*3l| z20k&Iy#JOMCm!SME!#*27Up7K;#OGGpiaHh!Wo{uA8}@9?i(aai1VulBr0GODxJfL z4aS4+mXFMmnS7wEy9jR{IDjf)f@pJ}Jp(Q>pzp=qLF~aTlC@lna%` zEWpG{fR|h2Ox6b4zAAbKn9}K?M z;LhoVw9Lr|mNL;8=vRQ5l1GRGe?GA<-%d9R3GqI^TZEsEt3Zyx4cxCZMj{WgyUEf? zx?<07dP4dt%)FxjF$N0E)mv#ez2G=?x~74gjsv*#nIhW8vM!h-@!&Gg1I3$uqQBDR z`WLZ1FfjIqx=GIk;mA;2UC;L6-^}N5cAY?p!&%s%7hnHH?iDF$u7Q)QlCh|Hm>i2( zfZzNIAbR#gCT@(M#9F@~{c=e-f5u~uGqWC6oCt#cnI+_TzymoVlr)D7eLur)&qSNV^FNqszf#=mK86wVvJY)MIM+G_bp10be2#VD&yXaBsRt z-81FjOdT6&Qpmvc6T>u0?;j_(?g+R(WPQKP2kN~uk!T+}i7zMlP-BW0M%A!I*utL7*=JRKk! zf5frocRc!E*+aL@cuN<^=woK)VsKrtjiZwDm+bf)gDQ_4z@dkaC=a{C+t69Wwb%?Q zo#J8N>cjZf_Xuc>|Dr~pb7+*&H{$)_FS8=&B-*UkhNZuS=m+$K6-}|EDVB9AD4CI+ zM-JjkGRY`tMWX-W?dDMldr^H!KizG>c3?z;$hBnF;g$EB+)GV@-G-lNto$g)b>%j4 z{+k`;(@m<&-XD(7SbkRDR8?H(7|nWvEbw5&0azMvn6vZS7IeQm4W^{ap-ELLF3V2G zh7n`d@w1Y*m~srAE|c+EP2v>Lu6OZ$R2 zW5237rU&=I)vep`!!=E)4vzqTr&M#kuP*pfCm34O!Z2!c0DXI$<(JHH#Bc8G94Awf z_@|9BW2LQ}@fp6rZ4*P;pG{1562Te8vM_nr)?vDBQlZlQJc;p(!(}fIppEopp#NE6Md>tHG&&7Z7jD7D*Y$B} ztPiL;X>eAX6@ncn6$4)<;_gE?Xu;`t()M*NI1UKlQd=eBrO3h4#_YV){5pNu9Ro$- zGC1@;oDA+|v#gaHLFch3nf{2)dEaHZhi8J!M-)S`Eam|GT#$o&N2UT_RRUZwVxO(y zW59Ppk>fh#K>zM#-=lhk(8b>OJ24eL4CT@?mT!_Y=N9z~h^L))Y1FxVK8~qvhtAOb zxO8BY^S8c?sD;{NW${|_{;?0H6?t7|5c#twG=w25(TCqQ)rR9KYqD&m1$ADG(w1H?Em5WC)5;hmNc_*OX;8(IltxoZ>K!Bt0r z79BL?+e6|vvpM;IN6b460om{25b{YD{8!pQ+l*vVp+8DjXyswpas|jS>md1zElTMm zQ_^&;encpiiWH=y=bRAwSyl)|RdVq$BMQA^3b<(PAvEpPhu+CGG)I&mpRpZ0e0vma zM^A#f!DHe#B#v!UJh1(fBJ4bv3LZ3!ShZ)8zb67Q!fi3yP1%QUcUzDI)kV-VB1v9{ zn1Z~OA7l?JzzgSnB<*<*%$&IkA8Y+&j@>Sy<_}1MgzY>*URe2a~-0cbT^i9b87oHgJ9E+{F2Z3>^g#~p;Ox6!2Yo0xNOo3k^jf$m<>##z}oNRIvZ1M|~% z;2*07Qo*aC{b^FTuRxgVQaG33`nAv)y9T)EhcQCeaQR(x{H5fFVpVO--Ji~MnExKuUL>1#1iDS-wD zv-icv18Ai>53~>7B8wuwFzViCV4;2wwN;vm{kL|2kMuP{KN-Qs?|Z3Anj+qQB8&cJ zo#w+Q?8$)YDpXEf1QvM*VLYG<>O+L-jcB%xIf=n&k~Wp||Yfut?Ai`3=$lOCQCIvtM# z>-+-Jt<**AcSxYc{csW(GeGfi9zJL1FiTo4a|WWPL-m;y@}O4-V~WP{#PzMv(diG? zoyR~$xf%+N93sa=_~H4K1W3_51J@3V!a58=b(=w|XCuV>uk|T}C5S>1%S4^0tVl=e z?6KE#03Mv!#GHx_1lQ3TFe+~*)1w_h`lkUW?o}dbKiEX2xdUc=y>=jXMT0!Fv#UG% z-z0rkwt@3y*F8Kt;}%)tDn%9^WHsA*S>XD?9H-4yVffcQ1B;iI#BbXTw5kna*~~Ag ziZ?sIWtr~FRgCHG6U$IC(+&BBqnJ7ubKK0%o=(5c1?hqy98rw|*QX~zS+$9LOA&yM ziCoww>k0b)2gurv2;4Bwl1fM{LCeZVl|#hjLooIUG~l*- zr|V|?2C{R8I(E#@D%7UJm{9`T%xTxX~nu zbP`a0pUxAHgXSmq;jmH}Zm-qACDWF`3C}F}`13c%??NFtr^4ooXBFe$M=?+koe5H_ z#xYPo7>uJsLFZK?xQeoVLu)~92D1%M-g|^n%M9pJ@nCu_OuV?OftG&n zBHanMNoeH(2;XUqLJv05z&G-kc`6L=j4VMNn_L)-n+|aFb)9!+06B7OJ$inw;`|Ig z0P7NS$;;~&j}om6L8Jf0+>>o1e;{{wF| zzXaVuH#Df;fqRFw;EjC_IY%G)9demr8t{6iH;}lT_hkY$AXiXJ&)UX^Dq%imE_*J^%`FK zD&hK9H|a_@HAr8fLe(BeL22Y0V#@N`_HTYhqHH$M43ARq%%9}wt&-;sIQ;+pqK9(~ zKhod&EaPd+0p6ZpK%!=Dpm%P*rE4}Az=w$zGCLrd;kZj;(Ofq;C9n{a6K{fz)>hin zl7@Dbt|0O$1FY|a(X#nXcqY`A<`{*e)#h2a7kseq6`LXUE~1)0I8;owA9iTVVd?A` z5`Ku7i}XmKfp{Hmjfmqc9T3I2Lv^TeBMiP=7N)Cxis?w69{i*k6qv<8lV2Q-dwyl4RSTGU9Feo^A^4Wen7+;G_Kn$M{k)xT{X38%xXK?4w!uo{x{KDexQI zURdD)w!0R%FovUQ5QxoELqKKwXkFqHcIULZepabJbkCCJ6?D3>{w=nHG({OTzb}V% z9%=CET|T}&Aw;z2i9*oN9NDAiGHhi>i9y@4j3t`6&W}9-=6- z$R3PNiSv>*G-1oJ2zo0{mODkMv$s#jEEqdW3@! za-rnT{yKV8kLA5o#+yr5TqYaXynwvFHUv+yoM3-#R8O*?M_k_1_hE%JUT7GY9n##W zmC>M@GR`ELHDl?+P?QKc45z9iu=@NSPT?6Nus^bzD;bmpNg>fV$phT+E{kef%5b+X z7U!IvbfS8v9Eq=MI@ztLjeo~Jf#`g9HmfL14G-j_hqpdRTy*c-hb z9$y(j|3ZE^+fhZ!*cz9@&sI#2RN$EpZi5+d>fFO+wa}LSl{oN*p;77)^PZ95-7@VU zJ-umA)RYGv!J$O_WHjqsn1H0(O47Meg$qK@$m&;D;d*Hob_(0WaigtJvS2lRtw)b=*wx8*zrpVH%bz5hX!Sw7C_O&88lM|OVY zIGuZJQ7J@D3PERuGgR6s@b-rG5f_8`u%g!hqeewQu z{7{?+rS`3GQT-zg_{z_#4v8YxqG#y%_jx?Gx_I{990$20TIAW(I;uCYgDC7zL*D6& zpuTSwFHlzoj8`wfxU=){v`G@Ps%#DAyi$OUN;&Ss$%|mM*B;vk?!rLz3u@2q1>|L9 zxf0<`RNzey<1>5*qvw9dbsOX1oAqUCw<-)JqD^=UL~p>K>U4N@iS>n4sYA5Oih5Vg zL%8T&D4Oim#6w0EWZ6(IQ)lxZ@+{aK)9P=KQ2!7oomYVO&4VmU+YMcdlz7Rf%OT`} z3O2Jl&A%bhbshdsq27S)^sSZP77d8-_S9S9z&rxKeyZY}x2M^D@)syxrq7)`H5X}^ z054)H>y*5@5G^i_5dR||;2E25sO~UB7ccdihESe)JCi6wDycw!g%Ir&X9~A%grb zm3b~pE$GyRVYp>Pj<%+5q1s>hP~gx;mW{s^{;7G8>+i>rsyM*G&*^mjp%z&3DHvY; zsvyTFEHUS&JF=VuJo#h`^j?p|>i2Er^rxjjW$w|foI&P!TP}1hxeh78tDs@x3w)3_ zft|wFamK#WnEHz!#>CHFLiIPy_u(h8TD}i+s9s41Lm6cwbkG zkb55n;R;D(j0KLeT=Y5IRjv!+g2y6knPm^(#MoZ8qzL!D={vArXpU~m<>bG@QZ#;9 z&nd}EhT69m@l}=s`KTTW3%{;I5A#$AF=06&bs|K@_B?1`R^o=Z`?3t{e`J8oXI^PE zgt^6O*te*gtUe|R3H-v?xM2!>`c%PD&1ZKXbHW%)@l$XnSRTWByqSgJhVc3%4?=d> zQ~CZ(crfe(H50;4vKsJ_hR$t^K5OM{vp1bR`z7RsC~dHi#)lHFQipcx&CD{bwfY;RZV5jIGTHWVQv-fEN-?<(lOeOI`(g2fYJr%C@eKfy0-ibNq zyUCW?T+m(Ni^uDwA;ZH5m92My3(KVcdOaAeB7zwQ)mUnO%n;i4EJdxlkLEfr^|`tE zGAObm8sj8AVC=tM=F+@bDCj5%|5h}>lFA0$Ahm^MZ+xIrnNrdOro`{xb(*rz826s+ zW+L50!OmeR^nE&k6Pvz+r@?iyOjn!MKbV6z+Op~4$Y9Qt)Lp#2ZJmU_>mAYi&{f}f zw-lO9^Kt0HE=ZUqL}mZ#LqW}ANV)qJUwQkZpJ^&|%qW8AM4Zc6a|`vn*3dn1Jeu@~ zbz#iDh+nGtxKk}A^46cf@FM0#>0k z9(dVJ9lbhn&wp#_h3H-IXQY=7sImD%>lxglby0NWeLp>7&9dKQq`2JqGpS*eC8%8H zL;JE){8sxMx3Rmuwq7IpW^o4oHp~U*yJv7CapoL=2pAc2qSBssz(hF~(!^h|yYDNg zt1SbAt4bhmiZ&d&WPoiEXCdcW4;G$T4vBt&kiG0x{iRQj=|*>BbhiCU+tSqW=#UDQ zls(~OWvYOq-F4=e{yTVhtpfIT#bD1@e=rCuhvTFFs4FTlp?xtuS>XQZSLmx1yK(IS3DlLD3YT5~k}rF&!_mdh>E8lja`wt> z-uj*dkeoaTW_!)?`nUqvlm&2Pxe81}Pc+eE_rGHelW@4QpVJ!ukhHIP#6DAH5X;_m zZBA*y^Bg{uIqrZ7Wk$GtvJ|#ky`?j29kJ#1794hohk52=2;VO+^ZPYiIFv0}jCXBq#K@?PFCXI>-+U zhBQYXXf~XLH(qmG`kUoUnGBb2$U8|7jPUSZ)mKKMa}}}*Uc@rqle*j9;TW5f`GOXs zIHe{S^ELfR#Hkw`JBWL$WSfG55n_*c42%>XBrT5=XjXd|-nhfL;D4LYhCK#+>reZ! zM!^@BHdGQNy?*#nFdY(K&cmbL;{4}UpP6546LGeW7kSj5PUm(Blgn*lm?$-$daPWG zUAKE-%13XQ(yfVnS8I4tw+~{)-_Q~5L-eR)ABiiuib_dSdHfe$be(kzk@cQN>}>>i zM_cVcg|AG1cZXm%`vDFN@5PhWLG_I*PySBZ+OJoSVgh`*&C4(|LE;q)W+=8zf3Xu?iF&%8^?66Wxh< z^hKZzgcAvf2+E^FQHh+VMG~Gw6~Svu6Y@dp583MPgcnri^U4<8AX}~o@=x4&11k?~ zB{{R@IJf$3NclH`|3!U_T`=b*IiOewe9;@&x=9(YJL@1n?ljCUy#jO7?cnjIQ8?t; zf=!p(*-2#`#4NFhot66tL+=Q~pKKXkRe?Bv<3wp#ZxltExh&bIQa>!(YD#1}xU1Q zH-X6PXArA$6&3}==oe(|#?4FPnFU^M3vxs5WOdMx( z1xw+f6pJfNbJ@H82XXF|1ZKJLOIp?#P1HpuQ#WOKw9w1JTwQYr<$nRm$_9Gx+<#>K zHqKG)DFeZt^>llQ0{P744bI|%rfOA7IL9$^^lx>6;o`|K#5)K_^s`X1z68RoDOesW z<=7)vV3&dg%rUG3(W4xP^o4_bs7S#u$cVL|NzXah7y^CA?jx5R`rqaz#R# zHo2T`v^cv1r7AihL)nSj$2-AiuOqm3#zA;o9nY~Op5c*wJ~S#=1e(HZ8I`^eFj;vJ zC-Bl41MyqbzH$+cS-xlIezK?ATB70h%Y#(f=qr@&?IywF0yrL;LjJ2LhU%j?NJB{| zJX$qIoq0UG8FLOFTuI^XJI!RT{ViA~HxV^8x|v@#?W83r8E#mYHP{}!3BPA^oOt;n z_;R=w;sWlG2^OL}pB0pReH*TQUB|k@pOV&mr+4eYo}WFMMNBgvHyxL&cQujiV<*sgw9cM(c4K zkx!6^1y=#jrRdO?QDLZGaa6vyiG*$rhNC;Qxc8GJcE~5e#PyfZ&psMTb^g#Pl3wJp z%nR^nXV~QR`7jo5gvavy*qE~#_s=TjhYWk~KikdjXCM+=rx5ZnnuTMsqnc zye)DX{%Cf;xsJn24yuynkg1q~#5!4Agsj%`D+F=+9C-$V% zRW+9&I(iW}maQc^<^Sj!dtdB0!m)n8g;GapRmQo!n|*RE3O+|n!jTX++!jB`zRO%e zj+Z~d2ifNA=}q&|FLfAe@^ndp*ERAkYCoA_w;didWz+MOZB$a>H8Z(*3%_&93DjO! z&@>@(E`Qk}Pw4NNgkF(##L&T!<35Cg(iBB_mbsX;t7fq7e+Ox9^-469s3te3TH-sq z#n^UdIVN6lhb7)O>8b~uqw>Wqda`aawuMZFsS~xxb6yD(WXJU@BEI9m#pzh$eE~z| zFGB^#wscv)o)$k+2Ziba=y)xGKQ2C@&)&B|LpR0+{Att z-idqvX_CQ$)znT~7!(gq0-;?Dk-cRPYHA{^@z7P^e;jKO4^3UEOl^xEaJ)2Q zK8FHEJL@&9oahvOWN-&~-cH10Tc^`k>+12$^m}ykYYBEDx5sCX#!zNDKuY{nv=o%& zk2%)E=Wjn)4SPl8JEW37_bOx@}?!9R$zM2b1Lg{2)8YMjLDjB zFrs~sSXrfj+@Da0R0~02w*yU+b)Qq|OQ-Q}#9f?vFO}5fB+~7s26VNYFaF2>1*O?9 z@!{hfOc@ix!`foFMa~Np10+a6yEsOe1;eHwZ<6FJj00MKz*@zDzEhP%)857C|1%H5 z>zfmiQjM2_uB#3r6A9^u&D!QuJvx5*Sw1*h!_>$1iknn<)48w|R!+1vz-gD1kehAL0nFl}t!Eh3f?6 z_*tu4uVNjwp*mu>T<4tc?a`0uwQ&oI7;hV6a`lC3*YiCG1Tvxh)BEs8An3Zn^g zCi9lgF(i3cE#b2^4=p(EV7L2MqRqccto4OJZkrCKZj5FV?&Nh4yd^5@ zMv>{qKG2a_KZ&PmBb1qbAU4l#(Tgc3kS_egs%vlH^8O|G_*p+RcLYmvVdN2c=<%O}=T~{d%N3e50_S{RN*A0rv^{G}Ex^O<8^?L-otZ)?l^MwQmMPiV}StPMquxNP%cHDdd z{4q)X(>Fyl(BLReP=1PfidD3KA;AW(-yk zBcDjvyR#7X{yK|KpJ(B{(zTF%SAb{xD+QFtM!5UfA13nLIDOudNQ+jE5TSF!WNF}5 zm@!$F`CGA`&WudKUFHfri_-%*d6z6%V|ST0-PM9Aw+wjfNh#j(WlyP-Lo`(6zJmP= zwi^sNZNX);H^SS<;|AILO3`n?9ScQ{fOAv?JlpXD7Cc!1@0+?=iBO8mYDIwbCDEc| z$Drv-7=B;$fz0FB!97KpAbZIOT8X+Wv|8GGgt%dXU#D;{}rx~Ph(e>e#F_V z7IpFAo}YD3k>(FT&qIG$BfpYH8CRk2>%){T=|qj4qr@S2J+8?225&P1dQHm^f@Ad| zvP%MVw{F9Nr4=O4Ru+~OoWt~)*)WNU^3^=HaV(P)3?&)(uPF=XnMm@NY@f(KQkRJx zY0nvPE{DxaPRGrzZ}H9_5&lbV2W8OPKyUpz1n13*>5QRLJm0+2`*2!a%;X|(*c3ubcuV5>dTITrsM-Ve)8TI)3hgFGnq z)?|~X{{TP!z6(l1(}8Kt!gINkpl3oK=Ip#{AjZwXO44s}uIx@QzpX*G^lXQ+YlG-L zaTZQgpTYki6-Me``9R8F8@^~nES_Bd4|lCy33;b)vJ1N%z+9`DhW_|M*T)vYNt=@> z?06A2hnApy+jaE%_7C=LXk{G4Zj|S%6YJiXpo|w*xL+Nf4g3ras(5VFcY}iEi)EE;ev**s%`Ow~V8{~S{LhG0L zw6kRngqxqB8z=mOGGPx|m_3cxdUGz!a1v&n@~1-R76uIjLqN6K5EL!Fa2C76Ag*i( z3nPSZ%|-#_U3JA*tIx5rViQ67(l>Iq^C+3IwTj%9;QANkR;W2I6%0mGiT(Os#%wGO zhSf)@n+Kn6ITgl!DBVZ1-!5j@yXrilt0%zWN-Q%J<3tC3^@EJj4s4lqozBtYSfRW` z@W`r#-==A>*=08ARCz;}6%BA_t11|-yTr&OrIK&mznP5%qnOc`L;w971zmcMn+Yo+-FUjh$VXI!lyOE>?T0KV1@SWzPg za$DYFzdw(fI7))=;RNh|euvy9-wZ@Xq#CHjgmdMXR|i zR{?aUGdQv-k*+ur!kzg#lrQHF=X?pYH}zsrSrvJ|*q(}}hrzKvJGhcr2sJap*po)w z*>@`mhB%h+V2m)BWgbDJ7miGv(RTQ+>?}O8Ur0roM8Hh_8r|!rNMeqxrA<~2VCk+# zeovf?ABYu1{p8rD8Cqv_)17-Dn;jT$Kbfda9U< z;~Q~*ZY8OnP=IF_93msTXRt^|%;hHiJ7+1h zeu{>f-I>h4MRmC3c>yW=nn|x#s=@nAKQeWG5xF)&nmepG9jAF9dR={M>L$v45 zX(I6ABu=|-0rAPlVM(tSd^9V9wo`6k_D+%5zb*#n&9ks|p#*l`@x)iJWbt+RC8}gm zLUo^TE~QC+u;5hzw5>0Os?eEGmwSzMn^;QjKg`2LA>N$BDgq9OOhp_oge`VM^l60w zq@Wl&Z8w1QG#NTNF$~XaK7gk`&w)En!$D!jDk$-tjF*xRFwcH5;IO)ZRP4`$P8lKi zX_AW5xo(wLb~NtV;fu*ryT~2Q87Sgd0wO=S{mFtHw4#gg*t&GEkg-Q)oztM2-zA@7Qkb3xLu@Lq1QSK3LSp!QXuiFI?YVmj?HjDgoX6>GVT~2t7`FnvmSn=n zjU(9F_l7RXD}$+pp;#Duk|dvYqz}8!l2~rmqc+PH0-gxqpq@Xg=_rm$T*sxy`WlJ3 zW)RV~Yp8o@hq+eUXrD zwuFQ0D0qG=+NV9`SYFsx!J(=qL<1%e;t>^|jK#i-8 zpu~MUPyI-R@}hN^=q3kqrm3>8gZI&Yleq4i=4n`v!0o$r=g_kQad2ae3phWS1%LNe zk_D>zoY$=oE~k{>`h7fTK5Isj8#z9U3)ktpEewaZO3*#3e(-5H4w@7Gu|{tbU_+%d zzP~;N4|T^N|91dtxGNB&O9z?4+3jR|OEMd8b(4Ap#bUP2au8#jVc+v8)>7IJM;wyz z7qg$vSlUKaGUHKf(L!oO_rvQ|Ye2rh4+IPhnhJC5KyN&j?(W?V$<{}ReEEJbxHSjf zuMoj~QIlzv|3)~x%NsuZ;JT`Rm0<48ViX?krL3YeR-!hRuTmp2{wGlKBFBT-oDRo? zgW>O@EwJ8Q1Zc)7j(K^WNGo`trCAfRTk!!AHtwTqrwHJ|8eNRwyI``!cd{V0nLd!* zk0!JXGiRQ}Q(rSROqQ!gi{$u&{QZHPuZp8|MxhpsKVu;Uog*h zq+(s6I*M$H10!P-(k$G_E=|1FI9-$v2c8XZKPTskfBg`N`~^~KqQ<%MqRAJ-kFZUx z3@jH0kcICS!-JQXF|GL@6;T)?Qw>$&$J{NjZ`)7!TUpAmIRdCN{GIOb830q>M84C! zQ?NKz2h4Q~L9f{kTU}p(q2pXAlH>La*<7x7Z~|Pk@$VJ+qk%J6q~@VKD|Pi^Il3FO=>-NJh&iVxVdT-s70i>bpZ= znB~LI)gxs1=@XpvFBG4=i$lG?y5Rh*3{=&RL*T<*c=$vXQ8dye-Shh4PM#!8{#!?! zp0vX1hB7K$6^c0*Hb7I0F5PSOkjPr~!2QZuIOe1a4jiK+a_qmx70fyK9bk&}p#%8# zWHPy`)`$;(Z@>hfGAP*jlD`!K9SpXrp=<7PR35D%ORqk|hv#mR&(|sD2kD?= z&{XJ}x&WWc2H`;6RHzH#x-GpL@bro%c;;_I>p#1}DqkIPxsKGa&TZ)MwHD-d@WH@- z0jcoqqxN$%Q00gftUYmsTnbL*ax#(7w9W@xKbK=>iY6=#m;u47q9EA58kA$+)6-?C z@MZE({qOGasL;W~G5I}gjz?<4 z5e|M!EVFKshxS^yRQdxw`PTrZXKBOdyP9Zjf0TSl3?xUM#W8`xrjYV;J2*KdK-f?e zuFg|75G|jN*&bQs#y}-neX^mNDJG!nH%MZpFm&m;PIB&Z5VSFaXd~m#|{5CobQS4cm*~(6lBMo^MJ7Zb+Sq1vlO^fm?(btL>kup^Xw6 z+o|$BF5H11k4nirsZ<=hT1m{d7NXFYFn>6p6jCZIP$kC&n8y7eldV&KM#_@79F^cD z%X3Vb7eDF2OEFMp^O&5RwjN9OF9U@~+^jwzg{kbC#d&T*p}0C7mBV6TK;tDNa_I)q zZyg|ElmCO|g&vU2ot@P-0#s9K6()olqoaQ@>=5zAhKHFT-SCE)wE8lWRN@2OUVoTl z4+inTyG39lump<6CPRr?KG<#zgWp0ipxCeutR&>%eVHE)4D`Zdd#;Z-w3O`2JVT;1 zC5d4WfqkLpVUouo?mj&cmOb<5cn^{W_7fK1;%kCf+0zE=THnCfS|j`+e2qv~?_<+` za5IELOFX2!mzi0ihC(Y7@S|J-xA)NmDZguAS3D6{@On__;#7#+83^k}E%3q{UHXpX zquY z0d-x2n4ga(@G~NnNXn%zbcsw17#MR-4i{N0<#C+;&`)~lV}?|fx0}r0n9Ow~`QvDlh1LwlfwEsX?GYLOb7~s;WPjQIcBKP{N zVc8cC8fph1NK2XAm5mJj@(lRX|Tlm!rEY$3x`&=F8pk!E(!5jFju7 zIhVW8%a_9Vz&qlk&oQUl0`Z6HbiUqieT;u;hEr_r!foiJ8yEeDt=R{`+Wk6RcJn3j zZ{MU3|7$^hWfYbuwXlANE`UX(D?AXMh4)Ol$=4n8K)qf7wk1@P!iSc4LTxp^s`G^p zAcxaupaIWkJ;#om%pYDDOfoiSz!d`l81;#PxqMA>oR^JHR+m9yWC$En*o>!>xNeb> z6#eqy9Y#IcN?*j4vWEL+g6K4JWUoAjp|gz!zluK)ktJ{8UFjR5eRuzp>HHEhwh7%*R`E6}Jc;JdU_!D;& zI`NK}-QEF{t}Ud|yRU+&$wT<&BFpz~PQWcK-uS`07pvbYH+5#Q06EEUkd5S=Q(W#8 zpV6=TzM`tAK8P-ukBUDhk<4l>9M7|ZFXn$p%z_qFJm*ci_RR%oFh|XVC`kV0fQ2r< zY1iR$%DSAuy@`G(8YGJcr=3A#rj_jc@sobJo`gEjTd0EI9xU$vK%LjuLPbgud9%&~ zo32mbM|0ln13&WdhjKAE`liA`ULAch$qWupy$#LK0Z$+8h20PSl8D%K7_%>#y>BXp z{9plo+mUpp*I1Vf?$d^H+pnNzp-GKuhG|9LYo_?P9eX3^7S^Q(puBM+6kdx&^!!Hs z4su<8he%e1T%d1XKBjL!SmA|Fp=5={U-rzrh191$7qrV3!Tf7+IKk13R@huZw>AlW z#sz<3DJDwl{4%H>)d8_no2h@F2;aZOn4O?!j}lky4E!VhQEjO#5}SGpTQj$SJIAwU z-=9Hg*HlnAJDtSmjnl#ZL^*!=L--jX!vC{c6Jyu1sH3zLs|t4EVuxDPwoQN!TvjCI z{0L*=lnbd}X5;&362!JWo*XnY8-t?WkU;|s9Q-#yL4gkp#;1jc7}Gj zJ~t2%Z*P*gT+K05LUC7S6}jYi0xphDMSVX@uzzxrzMJ)#PRq|DY5G!d|N3;y3wQx` z&n59^)h1Bjz8NLKvT!>r5#%3!2b%@vsBC-%J#Kx#EZsXm@2Qd(7PcVnkp-U)Hxl*J z3ec1Zboa*?vh3wX42zz_H$T0DZ22GzvnoO$aK(1mrRf2>hi#$4q@ij4>l!>RlM5Y6 zV!(Tlg~^wmf$)_V)Ne!&{*6w75%0&0iIF_)7F`9!pWWC6#X=Y}BE&1%xD^~%21CU5 zPxzmsI{%ls8C zV^?gjWA;2s!-XHC$xvWEv~JpoN&OiduX`!juL(gd&s^s6HEI6RWFZvUs|sFG^GWR< zdl+#pA!qgm!PhDM;3hE(w+&0dA!-cvw_cM_k6auWd;}%KU7#M93=h2Yz}}*Qgjny! z_p|?_i#;8%F4F@Mc`eCQJ#P? z`KwUkn;COe?g46)1v5__CCJO9aPnutCX7fBML~xT7#-9IyGE`s7vJfDScWBAsBcX= zCr#!{2_>NQlqM=nR)bdiza}~JrL=UU5dA#+U^-tHwl31fQ0;0O)%guWCM;}Pp&d;X zxm@Gu)_h{S#}s_a{Arn^7Ys&q(7!`spv`&Xj)rv-!Idc(^Foj(<9(Z$3`*nBZv{vk z_ku&i5nz3;fXne~W8&O!n(E$1H7h?7qnbc+rtSv~_m+p6>)wq1p$WXvAwt3(yx=~m z1jA-4Qlk45TE-h7;MzKvv8@MvEyTft^KyTmd!DRbzZYk8{G>x@4lem0KzaNOJosq? zGr7OtGG!66Xu(;U;q-~g-7E?p65{aQpCx<&1EluC6CwT@w}(&s0#*w1@b>LoxOCBh zMoSccu8|rlKJ~=0_lcmU{hD}<_@e63m$dEKGP-ghmmhYS1wGtMZpFnu{1XvRR2pZ& zRv90rbu1I&nP3brcUkji=5nlln`ymujg-pcKu zCtO4%W@;gA-hpOy8*uHcYjAoj2da!i>9j@*@_N7oYbJf9?Uhn|S0^=&Lw*vh_qUSJ zSdKS%-WfE*qw)E!FKlR#D-qs%3d^@-(_b@x(z$=7(L##fM0cE~w$%e5_h>2%IG$ly zeJ>jNwgg`7K0qY2{o$mrB|J&#K<~q1e1SjT*dy8r;GDi3Ztt1F41P&rcB|Ili3MLF zH7x_b#{VEY{O>`s#tOWDKbijIzGdapjzgFCeZ0HZg5S2r9VJ^zKxp?3++OsR47ukK zg}IN}W&7%p8P=oj+bPU)UV#D46}UB03MTy6PAsb=Q1E~S4EmkInu!ra%t4pj$mU#h zT%O?F_%@vAI}Z&6&n2#rq?fa_K3$=pH8 zGel)4P4ua3z)`b17=mTgL|6XEMEM27 zbY`G2p1k;k4fb71pM5?K3%XR9p*II`*`HXL+SEv%i}MifA0qQBB5?M^t5Dvv2>k4P zpe*YNkx_6Y0a7Uhi;lv_lyYD#Z;-2NeHpF259lO6V;cOKK^LzWPDC8QH1mJa!QSU6 zv~4CMa3c*Pm_c$_W(srVhaasWCt=FLDEvB;Pv==WGtW$Fu`p8~?bdR9*oCuUVD|<3 zY0CiT$r9$3)_LJ-?r$uDjIUBXjrxa8D(A{z_s_}vX@&Xh zgL{e4`}_fxZa+;{CFnAX9yE}zvYdnbVK~R*-2#E-U&x645c#^`4E);{0(wSWFrV{Q zsBem-M1|wmRyTm-pCU4rKbgNq*N`+XJ`NHQDfr*|POz)2qzwb@bTm8?Qk?YZoRECb zk!+&j|4pJ#_VmKTS_yuhL>$3XjxCutAI9|FHccEli4Jn1@HccOdD%AvI~EqPiwEK% zZNWDXUA>>|*<*(4O~E*_WgireYmncs-5`UTTejZ2fw^@TS*_Y4vLi_v@?I_k+t#Uc zU_RoCo3RF;P0k|2&8Qy?KOw6JqPQ-YEPc&ou}_;;p;hEgh|1o>WNm-Wu}fqyWpN=b zz--#6APsvjox&9hH9;g{A@ILEfVkdW@SCT=KdyETZPt!~?e{Zyv_=wCBu41RbbuSi z;>djIC$kF^u~WDk%I^B3#=>FJ?Yz?9*_Ge;s9k_B5@p2BH@yUHS-TnCZ!WM}yqLV_ zbN8d=5lr^&4@OFG^#V5wd|$}hdj@RR zp8@+_GeEIE5zl+4;=E=>JYXqnkZ@-WIcLuz{C!7r51l~MxApZ4?bo7LO%zODJ{$5v z&cOY-4D9t&gIj8y^p58~+<)mP*{L*@6y0jVN6*Zl^LY`p&;q9as5!~X)}Wskt;FYg znfOnIrBToSk%EFFcxFi(K8!9vMJ5-^1WG}Q&w1W-W`OJDkN9ujC#v{Sls;ae%b(&K z1K)32f@$Ux{8XGtwtK`8NtM4u^=2nF`kf|g7WmM#y$vY1QyrG7wLs1H0*ErY0B7^d zs6o|bXm6K>=!+MjT8(qhe$IvcN28dcAEzAqbPe-%3tiJ?r@NyIAZi$7Jmr9`M+*c4SRe_z{KDO0# zBCk$&4g^izfa=?-h!j&yFYq#8;N2+g`?m_(OLH;taDwCbA zKCoVP3e~<{PWolIdsu5a%3pDYa|i68@uMg|DP=m;onMFxl&%9`ECT|4*1_(unX9r(D>65Ai^ z* z7)0yt0~fzLM5C!663lbxC4WI;dAW+tS-1sfJ?O=Sg|CUa#w1qewJpxy-v-;3X5sC- z_vtUq9CS%zdy!82T6 z7>}pdU4yZ+w;-*h7WBj?z@|!lIO*z&PxyvVXEz7tC)$&fSXx-0xf{<|j?K`~~DE_d)-1S@=kVNJB7} z6Y^J~`fgEV@xvcXm5VabOq;|THv2_uZmpt=b7PVBNegba3G?c1KcX`Qyr{?4*YuUu zcYNm1h(R}+;A8kM&eL%bU&Ky;wx(j*7_3b+avLGDNeqK)TdBQ~GNk1t!OjiAcqDEg zgh3YjAbdVg_Z5*=Cm1(%9OqM)iW<#Qb; z@iZ$u{Adw=jd%p;tH|w_MW7fgXvJ6$q)u?b<~()y@oyr1*^+{tTyA1ZN-ezgu7L3u zs-UyIi<`F;!y?h|Oi1}g>~?9wLC!al`gRSDEA6Kl$6_dZXgbW#kH5df(3Ntx2K@(UcIe}tUBV3hpLzU(n*wh?OMDJz7Pv{2?xfNi( zA{LT{66h|jhnYUlj1GpSprULHh|b}%qC$b-YPkU=rSIU2upazbyck~Va$N#X5&qLs zC$>;m4E~%8$1St!h}>sSjOr5Q7e8}=Oz}TN!jaz2m`tM!E1dqb=s@ITln`K znf_6ags0j;|9pQmnji~S0-wo#Cu6MeUxz-b26%6?9Td*lhTpF#vZLDT;8a*7_+631 zC!)fbpjQN;PgF7U*Lm__nLOs7+D*T#EW+N@Ay9wCImbNzp=@C_sXTmv2C6%tlHxpc z>p}>hGJ)^3|giy!iJW5-2G@G*;#Bt`yZXezh$1V;NN?yBQ}71yx1hM zZ7#lFW{Lw`S8a0&kKGt$0y_J8XiugccyXSA_Mdl@@>O++Ie~wMzi`^NcnxjE9TsaS7tPatFq7=f< zuZH7C57Kj9J3(T$A5^6kF!CEt;WvFHTqLp$yz(dVZrD9T_q`&>7+9lnoe#vNP2(LH zGNGznr*ZIb7McD(%W{jCn8F^zS8!PhjZ=T}2df z`$Xc8$w1c1qiE@?2-~D&agXms66jX~E@3TLBPIYxDiT5Zmjk)qAA^!lg_$e21BqGV zDBW0#tXzXO7>QJ1=f(!q{ks)yyTj0MWG>7;ArA5)TfzP07CNbR4w9dDnA1u{u%PxG zstDH+hfh-6|8^2iS_=H#l#A6g+Jk#Q4Db#7edo*^2_4CvO>c-tuLS zS(Q=T)CsTOn`357GVT$}fqyT#Z)?gD2#~BKULX50#&0rSFwg+?j|F5_X)7ty$Ol7y z0IXC>N8hC(RR4S%T|K=OUz}P+Mn$+YNs!Btr0k&+lrJHEo(=y#Z^V^aP3W$W#!g9j zO{V|h&crbdtV~>M@Lnv9E_ynT|7S@MyXxt;MhCTRuwXnJUC%k;;JL@l!olCvP%{kc z4qIc-bdDX9cNt8oH^a=FiOhxn0+D~L?)20RaY12b{RRX(b*` z$iX9nj`VHjRftMV1=*xTVw277>Yr`HX$5KY(_L-QFQedc=?mE=--EZfbGas2lvsqn zBypzOh{*GBJP=s|d&M_EM3Wyp@z{a;=0xGU8&^r4y$vK>7NK&zw>hWhcT_4Z0^L&J zb`SMjK4u2kyx$6s&wHcpn>{4GZ6*xw-Us&|#KG3ZvtgH6I^H%2CUbD z16h6aUydRad8+ZSbzb1}0KLqGrjCzB*=q&J%VLzQmafcR=;JV&3XXl{t zB^7uTlLUkOWISe-fawRqiSb*m8}=;-6?`ShU*Sib*X%LqJvCqs3~LbaM?b0JtsgY4 z)(!Xk6XpxKB-5d|i|{<3YdQaFA=j&yFjw2c=&)%Xo%?+!Y~B4>wuqBIZWCiO%68ZLW8Cf-=_LH9dc_2ledH9fXF2@H4^6kS0RKw zH9}<5?LJgGcm&Cu946TEJ$UI~hDr0eS-rdmjTOzKj{+C)ysqfcXC4{2OIDP(DJ~ct zx=QHB>3^BK-%q2&Tz!npO9P`AIfxpaMM4ro@cQj!C>iaf?rsry(C`>8oze_DmfwaK zfg@yT`w{S!(Im6(579r|8&d12Hf+h}JoD?Vpze(XuS@(TZk#=lKf%oe&&t%`rvb^UgC=KK{wQubLXJ=`cbfk*N#qMRMC6189 zNAV!ByaJcb>7^HA>(G_stM?~cfptSHoK!Kzp8mOf5XrGN*`Q7mOl_0&0T2Qg|IvwM_&v)_? zAvm%Y$Ag-&qKD%tCv_34tH!W@$2`1b)kk8NB*Ula4{-Ryb*%eQMavUwvHo2cHtqU@ zB_7MrI^UNx#MI(Vsfn!nj$m>j6c$9P($?%e@So*r)5!JR> z0$JLwAQ*ZUwEvUEf?DqWvq=+Q2@F8f*fN-=!sWMPm!NgRAv(WE2L$dI;6x2G5INyP zgXkT$zNLvu&EANy`)c5LfDbM$uVPp0=8%Z@D`{rQK5%V_X;PZDnSbPN3m!8TrPq`j za8Y*&Q2sK5@Wmo**4G>~8>jSqp$e_%=6k%zlej~>iB>4*H(7l&03{V~+O`>(*IZV= zv*bLDsWZcv6fP5VsC60p;ruVw_g5LtS@#-( z`*yJtpU>nywm46IMp?t2PdgmcIs#!b=jg_!Q_QPkr1IZI>Do}}vXyP(#QPo%zV zrGY9Jh<55zvODBHF}YufCob54`Y}pQO1NO%sU4sZF97Oc#x&gGAsXgv!6k2`=ng*t-rX*}q8oW_l$#m-wT5-J+XKB~K{rLH` zhwDnLh)zL+SZk^;mJc@RRrH?4dE9Nb9`i!iz*a{+FyQW_J}bFw!)8T(l~@o;jCY}S z)*#0po=zSpeMQ$pTnG5)O=`oj4z3D%6H7bJLGjQS>U0KaxhxAtN|Crobpj9XNYXMh z&LyeN^)_atfKZtXFKki--SJNbzx4%!&hirS{_k{NiI*CN?7agKue;I2rXHp1edr18 zG?@KyB1)}PNBuM0JTl-N6*%^ZoKY459XB;{K-3B!K7Rzs~#xy8g{Z8;VEq5$Qr1ZcH&620*NdVJenxbZrf6#IUn6WKF3 zQZ@!lS0s`r1#1X#PQzxy{ZRTcl!(-ap+cbsY;8p#K z58^1m^&^gpgmL@nIxgzPdf94`3!bs9JMQ4h1Ma=74FO+f~FQ4zEWa##-k! z6|89Ak1s}8II%Tkz~uZW?J}l8(y`m z!G{e=G_^<-d|H=*U!O5X?#qWT*;d?YwVJlyDyH*)$Km_R6|m|<0(=d7LRzFaX5a+} zTge-CG%XfoWvM^vCRI+j%x(>3PupYXRMwhZak`}0 z1YW#C4HS;GuIOJppz8)JH`4Fnvj#mcu?aQ$u~9!(x*gU3D* z@pe}1<{S(jYP7~8W#uUlRPCYW=4z2)8?C7aiPrDrm}czd1X z?mCLU3f94zbGEprEDn3tM=-83FX@*t1~s04CTV)+IA~}?KP^+?y!W11-fl-Eo7^x^ zM-=D$m&M!-+lX2vcj@p`KU97mi2@e$i2SxDdNDg5?FA2!e}2Vu@b)hnqoRs8j%|j) z(j&|_`+0CT;sq6m`p#TclmOe_RQNk{C-lx=gn?Q`u)aNk_*S36ZxLE>P9qfQdJ7bu za)t;To}@Q^b2;kuxzN#2O%M>6-B_;5qs>u5p!QrF|8Cs{gUVZJb|jyWMfb?-u{mIC znG3Rc`_ajNFV1bVxhU!B?K_68S}#pKpxQo%uk`03z;W9`*Z%Hzm_91e4 zXua<__}JP3Cn{CI?k`bz<#Q=%C@#P~Yv+?U_m6D32W>DtKy(05}v@<6@dd znM0<~ezyVC-AZu6`xEq!*9Z}-<1)@s(Nt&H4djUg_H2nmU)fBIS)Kx4|8cC($>~75 zgz?q)5Nx($Kw;f72${Ku6?z{?jec~H&-DVdaXbZ&HOT>PO2Cs&wm_0X4RrS8)77RM zVb_H!Vm?$08!k-beq{|Xg`R`l-ZR*6IRndwe$xP(ZSYWHKFl+_Pp^9QQkz?z+*|7u zW1}`34c(h@yBNWt9lczB-~e=`O~uT(8${E19(nZ~VckxyOD&RvO>auTc2NPk?cf-h z9l~Vvi7B4B9s!fn{zuVy_*415aokQ4lD!k5G*mLqeVvMkP@$vO%|$yobT8$U(gAgh-!4!*G4G-!)F zF>;Outu6UvYFRY3sW<{p7gUoI8g-;h=@oJ7>Y^d%*bvvaA0)1^VL``p(&H`1_)XVR zJ$nx<4TvCn@GWgwt%Q7M#DKLmA0F=;fJ-?{|FQi&e29%hI(3aKn{kb!a@m(k`|u*) z!BjF)H3MJ+++n0D)yP_)I4X{Y~d!$f+(k`kmoM&lbeE?(gv6?g6rPWEPw} ztAWav&&cpmW6m|b2<+CEqqXVxG3J~-%OI%~E=Eh>nA%m6JJ*6^wpAOw7fZtg!{_tZ z6AM1)<%!`#XOOX;$7B)vU`3uHbei6vK3dwatvnE#b!Wnx-)lKZntZ4PD`u z!7#RLtBh7u1Vi^r7nbI1hWX_&!g_l=6qkpU!nTYyva<0qC~1vR!Cf>*vXAM?*!$^uL**DEFM)!?Dc=;*N*!KpG2mLX2 zRei~ctnDK=gS_x0??R|f%7pjr%Bc3plx4JF7DUUhz{f2sAkpG3O;Xwl8NrSaU>1re z?XKXzPo5a^p1EuN3ZP%UiI85-5GOZP1IGW;f*p5)p-YX2hKzRu|9_z{GwLY1%g5Bt zj-Ch3YzruSHHJUORarFkH0B&K!#baT#HW^F#OpJAd>sPZiE(=nuVjIN#1<^y?JtmpcPGm*ilkvUJ;N`JFd=e6gRkA~P`h``@crntAKFo`RgG|OOu<0j$a{A3r5mhYk`WDB7p{u`$-iv_zjpCY%KU2(rf zCfw37f{Xu^!mG_HxY+w0EB~lE`FG$qEzuMKSrOyWt(LAunBe>c1D%Mz(qFiJnT{g?(CG=q%4J4tX3 z2U+J9^dGDxNqLv>3bSh$xhxkv66doTuJF-EEls5RvUT;)uo*0#{050H&Y|m=Fxy6* z=^!7>gk#$ZF#gDWs54pu9|R2W`Wl-wDd*$x!)?k2|3? zKv-JLv-ykyGxN8@aJPAosAq+dxi6@f$6w-}xD>q)Y=woMdq|PTEXJX|AHO@kh8YF= zAZ&OO_8s>l3j6!$j#~<_A|Qb*hH!Zj zt(M*)OWst~u2-6*i=4%|!MmzSx#~Cav#*Wlum28QjbC&Vl+ZH{$>1*SO!B;Uz>>RD z#>wNy@xq}NEM52-#|t0OTmcSa-ExLQURyt|0f{RVJRiRm--JY(+W_P}jnF}*=q%5CVPeV*=| z$rHVxyzd+NucaQ0wd}#jiE%aQ=0WU&8Ej*(z4*Ci4Ssnvf=TOai8tpT9bPRCZ~oK~ z>0u+_p54tn3k2A2JFnA(>$x?ew-mWIb}uDQm^s0@h+0q&_JP128N4xTG45ZZ0F5TD zH1&!aG&@ZWk#}jVBB2y`Jxw8sdk4O^7=R>f(P8w%9 zsi4QN2~v{Z#q6qGg4bcsNTuaVjQHz7&F@~v<`>UsFSDER^i&mQw=z3tY2%=}!Jd>e z-j%&pff$>&4PP}cWp*`Mpjv~MUR_{-b@SKY`hZ+IXf>BxvMvH#8d7Nub7wjh@q)8b zIvmGaltJXF8m#tS4*G|bsr9HGt~r;A$E;1TroE8EdnSkW4>KNbMIl;qxtW#J707w) z(aJgd^Bm0d3*)3oCQ{k>9xDA%oI7+olxn|AVU-5i;`_tntXX0=Aa{lazRvr>TJqkM zvp`%O(-SHfX46~J_Ra;4@P&fetSe-rY&{yEt^vM9O|VF^1sx6+VA#PL)PKGiT)SU` z|E3a16>XtErnW=j*JgU6RGKt0o@K*U{@Se(sWd3Ji*{CuF?ZFwHBHeqU>3g)hp*0s zCs!G7#En*F7w{r|))kFT?8k`buQHxcf5vHQNlmj%$=hOQyqGr%{wFi27{kOgfBuIC ziavlL$rwDASOZFn4&cChioeaikjP^^*ln-M{g85oe%L5Po`o7>kp63$@UVhhx6vYl zH`KwoQU`9Io5NkSPXzDODS*DwJ8aSX2i4y8^yvyqu8yKE$$Xv!r=P{KtQudzXTMq8 z!mxte1ezT?@|0TS#mJoHxj_!}pfTa)k$&A(J zC|>Rl8&j*%N@W(52*;v3SkeWz1K>y>7ma%Qu#~w=*vmYi9kP+|vvnUn_w2&epPlgP zR0PvkQ3SSZIyrl+6mlC+LhUmbmM8JRvOCrIfZ-j}vYj~3VJjy0J|>DkzC)mVF0sz4 zM|W?2cFW~Xj4E-!l?M`FdRz|?@6u(nQpq8+y>y*>B`UO3!%6>zxV@D5j7HSarq|n` zsDb%i$sC{$nK?;8lpr~p8BE6<#&O4n6|`!ZF|4XMOJjK7;%?n6sy`-za{f)Q;_G#^ zm{$SW@2rT2nhLlkePb2>XOGF{w(zdXk$z(C6cerv_~+4DEM9vXB3lZ89l?UBd7i8z zHEP^+p6Msn z&;&}>&hYHq5dM~pW!3!2flaWXRzg&oCI(f2>~ag%=9JS|F%nJfK#^WrCR zfZe|B0(_n5pdSKLF^Khx6{EU~Y>;RADxPtme)&D5m~6$ZC4W$e{~{S1OvSN4Ta0gE z9N>T3q19cUts$63%H8!)c7oX>8%zKLS#5|a^MW}$_hQiBXwqx4ge0d5uy?+lWQD2^ zkn|UwxS})>?B6qv*V3zK`NkMu=1frs#RagZc?P>@M?F3K{wP_ie+2D1XTyJg@6iXF zOGq&p8}}j#i=InFH|IYZ3%QuVV6#6!Pk*G7fZIq-+&Kv}n`ee)On8R@McE!D7X_ zQHwOB)ebE-&L)pOG46tx8_@TXIjgmXD&`%a;EA;5$Uhz_LoOFmi8~$69hT4emcKfaWc4soXp_+P^oJ%1+&a?zbn&=n{XJ-}(iF z5@I>~cO1gl2r53;tYbC1i`p^S_Yj;?K?zSn<(X56Y};;3Fb=`$6O5bZ+9kAA_Ck8%AJu)G0KWE_bkB~zn6q~={Ab9i z?MYR^qrE=#{Q-c{X*(*>XifjyGX?CM^VK?|tFqq?q6W z9$9p{bCMj`vd1Xy+_pDdldoL&CU8Ah6;$plft-<229B<3buf3pe}q|}iqQxWnevH{k8y9%v8 zbx=H2lT=6Ez>q)7@#h{ps<+Jz#A*-lTEeQVs5c$U@xa zl!I+cdB7feIgckE;M0@KaC^QeNO<$obKI{$1w)JkcckOWF$p-gV}NK!KA=?*;$R#o zi4H+RY`yqK;D~2IvcPPx9d7`E=98#>_6Wwbhm&ZRQ@C(XO6{DRn}IUftl7PftoGxF zX?OQ&Xx*GaTzLDLepfk!=bpyp3!ZRVX10;;u8koYVA`~aUReB|TvGGJQ$wZjO-UQu_v>K8#&jrn{u~GX-XqjzkmV8Kg3iI7#^uAh z=-ggG?B;vXw5XNr-;d{k_}&w+digXN-g5!#zny{PDu(?nZ4TP^+`y|a3OD=PppKp? z6f>QQlZHLaPRB8FFrXd{3VBiVlsqigd5;;#hpC&{Co(TG77G{#Loee3j$Qu^a@>}3 zr-W9c9FuQ6eAx~5f07ta+C%!LUjm=U&w*J!DNsJ42d1xzK)~rPm8|lmD(|J?hRIVp z@PVI}x;#MxvCVMBil6q3X$dyRF)s=>441|+T$!lf|=peD%h+DLLK$z|Jd<+Vz?%=Am8X)^jk$tVE0F&;`LFr3|;1qI$Zg*dW^NoTa znZFcfy6j`Rzpm8eYY6S?d<6qc$9!*}GT4l4#q_2A2Fo3E}4iK?ilW_j^pn`@4ceBrPhkCA+ zi!UMmOFRm03PqF4afId218ry3VZ_M=@bHMFUtt}otFm2n$@`EMb3$z%GZSdty& zRL5jk>W~l;N$kb8z}4rOP_ohvHi=9@Utm5fSmq=uR^P=Z3Kz(Dx+78fdWPZTzbEhI z4$}L&&8*NLg5Yy-F8LWoIr=GHWTmSoWQBJE+g}$ol9!^|eO|VSi2ytAxgh)c#zm|h z-~H*B*E15k(4X}5%Cb!a&7q5&0{_Nn$i9{Y3-_PIfT?qEYaPSz<%>ZPo*P6b`6IG- ztbo`jg1GR;DLPfeIAObsK*3nrnD=xi-Egjsb$dk<dqr`0;Urm)xUi z=ROnuSuG#W0d%#B%knp}udCYM7wpuQ;tAGWLDBFmZ7IZYmx`q#tgh$N@=Crfn0xB0~)4T#JSiUxZ7R49hL+f31>f8+Q-|`L~*}9`frwkaHWMFS%IxOS1k-Q^8 zaA{c>`5qL7-M<)q@5BO#I(mT0RhX06QXwvj*jBSb8y}tfrN`u8t(a%r13wTLskZ8z7M_3G|o& z!w}Mvg6IcpiS7A~B{{VOexgRDtnk+cUEoDYBq>0Iz~UxzOKa}a#=j^RTOce3c~JXn3FlQaI? z6oslY;rLev)-f*QE3upd_t)P8|147odg6hBbEDwsoKa{t--@oc%E|Yv4`jDYGj^K_ zbJdogM801uF+kdbdt$t#)`nAqK}o4_dA>OKthtYVPI;i;_J_#kUqvU*W_ZvLif#ME zVVAuQtXG%AjQAFCY!iU@1$L06tgIS0Idftj;rSCd}9-IMcru`M0-8Lha+3h^4 zwIAh0mf?Q$MaBo({)0c$cgboKHQ3*zihujpp|7VRG`V*Xzj7~76fQwS=00WpBZ}U) z*+EmboFx1fowZ)0{Ls`BgXfp4!=FtR@UvuqN{#grVMPm6NSp@0!|T!2`Zr8wtcAo& zD%?XmcyM(=AXI(v$KO)GT_Qb144UDh^%*oVc~U+)K+lZsrytIN1q%D(u{?k90?8YODK z3P5tkdvfxa1nF$#MX{$W@{^fG57#&2wk;K?T0I3}x8tF-`Zx?}pFkPrJaTGA7mBh) zptn&Dl$Q&jI+K5Hx}|}s;uuuGTdc&3T8S8paky;`rqz*tnaHwNm1}_I#zcXHg-98ahi1(y$Q5Iehwl&AW}Gkn~R!6D>}dkEHYyQ~Bf12074U0+d{ZrZ>OQ5D0%nPH5lAsG6)LR>*G(e`WPqg_lA>@u-ArV%kcfysWsGO`^S(C9-_4? zc2-fP^_6s2t1@aj+tESyWw`7giNi;kI?ZA>;jFV6#+@=&!BAV)d$;bA^{{ql&{Yl}h35t_U)Vy3rNKs>rpj z4EjPh47~WdFtbVlU&zmehc>q0)Wu|GS;jzT330mxUU1aEFGQ`90^k?u1ylKBtj$0F zV`&-aqv$*n#CgLgscnVnVKpGyCJZiZdGJp7I^0a#3Z4ZWu=Q*Kq`jITK0<1!cyJrd z*i;Y4l$D^~b{%ef$IIRPfbpeQ`?A>Qld06d-<)-i#ObL$k>uL*E>2mgEH}zE4YbFi zpm&1=erjET2V_>j&0h-Ob9w=`n>bLb%VtJ}SCwgmRXjXer^Ws7m%Q7jd?H?-bhF}JcNp48U}V&C;~lHE|viuwDK zTs4!V8NU7G*F`?UJI%#w19{~9&1Void4=X&tHS?we)aD!}N2t7`J5iCgf?a z#P_)d+)Lg9xWzgg)x2ae@nHgddd6@W1K&_{%}aDk_g$!TG=rY^E5LUogt@CT&+Fnh zbR#>SzUteEQ!z8B`vXlfuVyBx-aJm-o=p>pxdSZ0o#N#D!zZ}PQ-b?Py_Uq7wnIhP ztm?KId5DT)wBRDeO^pvYJ!WAj7MH}ik#dvrB}%h1MT_9&&g3)l27M^gIc!a=bW9p16UIazG5K4zUpR%cwKCaX4rZIC4@4A(Hs&tzuDEQHh1d6bR@ka({go0u-{#j^dNy8SpDT)UE8FRRT~`Cv@Mv>eE?=Fc#F zViN8YY$L*blcaSAKc`8^1DryMjLyW^Kdb;Y0Xx>nhN{vV?nC zy#YBzMQD9~05uNYFutzG&o<|0_)ELH@w8qoSXs?SW4{pkA9Ih2kD5z0&Nm_ZeKCYe zm7s6vIRfzyptrP-!Q^=wG9YE^GM}|&D@wj%$cR> zHRdHMa81si!o>o0^pmnQ_Znv-nym1JT^B|vr+*=6>sEuQ?I!Gq`A$wW#Bla6X@-tQ zE=^F#M=`Y)&IN|oWU-2m9WTiAQZ7})WhNIBh$6K?PyWE;+b=Ld;|L6ED8ig`@pR+1 zBs_WRBm6!ZLQfo@&DN=s;imDbQ4_Za)Cjmme>R8Vt}U9t8*B@vTH5S4hUVa_G6}K! z{)7CelQjX)c7xWx7!1{}f$Gr+DC=9nuqFqX^gC|tDpI=2m$5t5#DKCQejl-J)$GUk7z zqGx4L^pZS2`OeGU;J5}vB!83eHJ9lb?+N^Q{5TwaZ9w}AR|4Db5VM1Io$7@uP<`!1 zT)zXsxVhmECev&>S70NSzi0YE?`LwmHXa~eT1oJWx0F0$yg}+F!^BngKl~oR^rSa4 zK9dxti(syauFb+IGo{SFUsQ>5ZolAk&MO#Pn88vqP@pNg(I`$CEzJF_(PBQQE~7WKR0Lu!Q|!^l%t zoIT_OVvLt?+Sn21MlXYP@?VhOZ=B<%tqZMxoS^3IArM?m$jWcYpjuT)s&`dkXTShh zbQn^$>3ozUyC|=g9eh;_!hJlJVE*I}Gs}*JO-Gv;7^D)`ZWac4(^Oob=0?t%2!ZCM zZ1Qy8R!U5FfUceflfBk*1ilsH&15YI));|LD?@NOliAJsUc(7nL-B=19sPFG2MzqP z@n)buOU331`7>&Yo)=4C595_8Z4boV>X&Hdhjzj(TaOoVG;nvhAp866$KaNjic;qy zLA`Ymb{_nM+qcf*u99au#XF;@yyikYE*_2^Gd8eP2Bh%pUm?azr3j|{Oybm4f-Tk; zInOuyz|ziC;(+@Av<`~GekYUj?yzF z8FcX9SB`f{56Z30#Ph`*a8#g-3G(wVp-KSntL2#~W1{(5nP}%+fQG9rb zK6UuX>HZu>*qhxzFq@EdB9BR*unsCOT!azz1bOv!7|*l--0mEtjm%Ddw*4H;*uMwI z@Ah(*TZZA~BOc_dk~h-3>ltV0Oc0&b1(UmuqD9(E=>47oOU??SoKvUqzqg%qmZ1@p zn(o92D;cQQ;X$)6OvlA)E!i^8OY2_;QOWEa<6`uK0Q-36^RJAtQUO?`Yecf&DsvQ=s%Kxqnizb%!*n*&Sl3mT!ZRRXVtTh!46k$IvXVohpbeARn4o=>Kvn zIi_ZTiTdlH;};Lk-V#MV3S7da`*qk}6(n zh~OLn1BNX<5q1@Zd2F$0b|#jtbD(B1jB{2zgX&(EgNnNW$Rk|_9~|8=B`*hS_q1vQ6k`8)REh1Ar_tm^JX%cKP(77$>R-u^CFgkA9cPciHEBhn zrfLeI1KIF~xo6IJG0gNW?*U(f08YmY(2W5FFyqcQ2oB98llxZT=G?oO&F*6PE5%Xn z#1Ui5Y!+->`hZSpEP%%oqo{l@6updvxFJ3g>@31K*#?%wlEFp9ujv?=uRKZ%eUo70 zgC!5NZ`W^s1Ki{sT@7hVWQ0u+kFXr7a{x=d;$wMM$^pa$}IjY ze)w##0WaOorP0pOH2zZrn3Tm)r)LbaOZ+3Jc9}iyS|bDj8l70&7>}?0Sm2#=npluB zte;U2nT8Uq4r5uA8{LL^1I!(dc^`IGE~YE>&!MDdDX1=f1n)Jka%PUVasHYW;OHtK zyb}U6pu4&@a;lxKIdYX)6-QB{TfCrYD@<35dxJ-@0f_X-kivP4e{_!}O#PSvPlrRm zKPZ~qj~2nw8hg0*MUnnw&i~$xJnS11!Q}h%7!X-Saj=sY$ggLd!Uxj$IZFkX?s-Uj z#=q0Uk=eMXgW+C<#*nR}QjEWzA8cQHund0%;;rqP7(e?8Scl9Z>x{26Tt_Bb+uRMu zweC_rsbZKFI|wX8A9T@|L!EsBXfh?py=Ya2-xJkoV%-qkBA`xYx7c8C$82=m5r?iB zJ-8}L5--m6CiB~hLBmmlRk`32{hQxNep$aof1Q51{*M)`XLfD9E-i!x>jFA&-D{}R z$l?@4%j3t!dvMyl6KH-k9_?NYx1x0D>Jdw<$Qyw>Hx#+Nxmx(w&Js1XYjD8P0w*3R zV#lZ;xUetaPGu`NzCHnUT;}6!m0ei#?hE`!%xUoANKSdf5LvrB0o=d70NK3xw9Kd7;$-;AvluR$#F6d3BPio`5Z;+e!{J{{Ca!1!j(Y?-R)bL}>3E)0 zdaa^M|HWbV$ZRTpv4*aZ55sEi2wm4|fo(cc^r?0ZE>L|4nPE%uKW7Q<;nfqAx0s;P zr^9q;em?lNsT1weTb|+PqGX8}RW&WeXLJap=ifr@9$t=X_XCnqpa|I-gE+FY z8T$VyGx?=1&iD}zL;g$Yr%g&Ev9TC}crF5dTFqWEbdaQ7(}k6$W!TR!`yS+`VDp@B zB>Ha*r$+t`DBM;cJl5e%FW7`SZ+3@@$%>kvnUd^}*Nzb@)(3jhS&A(<{0W;xgCS^P z9~CYzrZUM zRPZEE3!Ss-G~WNz02dz}C*PA&@j&)1`sw95R{MAYnlJEy_P=o;xmcW77A3;96X7_* zyeHdChT$?pQ1|!0jC+$_v-B#RI{?6jB6RG7xoY%`d}Ndoh8vtd&c!!ui&&oOqL z!L^ZfCj7>E_*#UIo86xR1N$Y>W^ov6f8_IA0R=R7Zd(nM*g^$sj}Jdfdvw zF5JI=Et63hl47SPwXVIbuxb1gXxxt@F$%9i=l52;bT<&EZ`z_|^A4)Y@bo6dLyel} ziE;oFxuVUT)ih*DCK^i&VysXej34nu=k94TZTSi=Pltjx?;7q^o?QIHGzNb9gcGxN zYvOp_18;SSaOVXHvjaUo!Nsi`urNFgQ7@9@w(@diZbh-S%hhvYPT1p`kTm?95L!u!f9 zIG^;^c(eUpa`b{Ykc%V>=2U|s&=wOvVXHQJo~|Q=?FXxUxYjeW$icSR=upzcvQ63y)%rbPzUL{++h#9W_v9C3v}waF5m_qL#|LkhccIfe zJ>;Y&!##s&GQD;+ezg-Ll|QQSD%E2>^O%RR<}CKy{yUIWH4NEz14tHcG+3M~1D7NV z?*5q*%pTZla{WRdzG79-oW}*k?x`lMW&fsJLl$d#$Q1v@_QU#ERrn<`ifuCC*tBCg zF|DEK*2ByGH|dA#Uo6FcGd{q-;coJHSs~r&JxYI<3xG`R72Ntu9gi`nvc5(Gc(zLd z4TLT6`7=w{P?3uomO-2%lWu$=(}*$+iTG%(0DDwZ6Q;>7@?U;8R$BPb=m*#7g_@aM z=^c{jy#6Prs#Ajd_g^~1@Ror*pC)8_$KlO4SBOujDt4shajK#NP&v8+`({{ybelNj z$C{vs<8DYf7mWwsb#Y>=C|LP!$MIKgczB5-o%p-~`ju-j{)-|+$Th*0%ipP`^;=L( zBII;kG}^uRj%ztbX#N8W9NM=XJOBF(x#7j2&XMP6dwF6ztIYVNc_l{0jT&v00(8*v zz-HY%DzYt{9N6^=r)U}WJU;{~S$?%fYt+H>%mT3NS7xh)^J3igDroptLAu_C(@g0A zcgBlp4fohqGYlQ3&-l4d}mLkS*=#&9O)PTx z%+T&7XPMvQ{VOu;7z-}BwcH=G4_BhAvJJkrUxksE1nJQ57W`!#34iX{;1=#4vJ)9U zZOc)Vp87}jL`74HTs7|bt!rp##Z6Sd6@)@d`G~ks7cF7Ug}}l27+=95J}Ya;#lCFn z(7ytT{oX--@m0K%Qj0+h_glZQ5Im1}lhspqsKyEx%v)t^{Kak(7p5`u=e{6v()0wC zwXNXThb5!SGHG~q*$4;n>#_HF5q3X`U>KURY>DJMD0Vyd`r^g=5Ythn;H7{3n12p+%8jVPIOc=nY-{lKG0i%R#*ACX2Mf&)u9z~{xD3} zD@`~isX*+fx;gOzH>iikX|UW>Ku;e%fVX42##fta6*)_hxN-xr3}>e)-<4L_i- zbQ4-N^KeX@9^=i4Vmc9N2_ok`(PYL5*%Pt{zWXyQpoa4tzFmc&nGgx`@ewGn%pa?j z8Q$A1N$k>KdcdU}5M~sE2gY2`Z#W9u=>+MINCpokqlkKZ9PYNegK#=O_mON8oVPNf zxfVz9s5^&_eTxM}wleU%N`!QQNQ{z`=gL%ZsAz6JXYao|tYhZ=~HF zvHxBgL~HesTD4y|@bLnTaN<&>bBnqA_P-|$el~c){|UyNw}5idd8|6wm06T+jz=g4gU&;_4Ac- z_eB}6)$)b9^o3}wl?#^7qX3?!5rgH;v|B0!-#MpX#x8O8<*ngRTB1k16PBP-ax~sK z!F*oUu)xiamu{3>2i1*VP>#vtRr%w^OO7n~2}1Tx zC}6(l&l~B`_v05035jE+4CrtaF3ZCc)q0w~;R;HLB!I?;n=sO~jkC=z8a#KM#j$0J z;88EZH5pfMjb{ltnXs7Lsegbxi-!TM;*c!jEs_f|v`iL^m_c784}FDveX`-99}^$;$7&C5Pg9{{Ij+k>uA0@|&v zAQh8mA^T?_M9<@6cC1SAbecchxRJwZtK%g%cHKi6=S&b=CxLNZlCWkS4@UfQg?Vu^ zz(@BWsD!rBL9s7XolhGane6{dyb4IZi8W5xF3PoV7sYuk3h-;!8jPH?678A1(F{h2b!nnMp3xd|>hYup_#?H(`CHc>OJ5(B-!EZ4l^RtxvYE|1}5Zd7ULTvo_I9ribvZOFeK!wGq?w*q`Q= zl6xAD7sd*+m0j_dXWv#o{o{MGanRDdM4BZLHgK zKaoX;lmQFB!0;S#(B7$x+S%7>NW?wDit57q?~0&*;}M2m5K2B+{3O0>cEFX$)ihPD z%{aq4o>pKYDH@bSpM{G+mAPXtavU=9GU*`Qo@b$W=o-AKDJP8Gh59+vgJ0Ed7`}7_ z!gl78I`(arZRHx6NL|IYW6P48?hBxQcRq9H$Uu#I6nRQFlMlnfpx&_s+*~F(xg$xq ze-i~M`PUq=3okh_QT+H)r2wC+8A0nP^Ilh=VCf`{>pNoT{+@dB^!hy9Tg*$HUY=z+ z-V$OjE)Qh5WI}B3Dqe1)V<|cOvmXnL9^(%2dXl;JG3-_RgClbpkMccN(DIp!noQQu z*;0@7CAZMx)EusZ&jh$jE#&4-@PfK7AKP-`E3R6$9Hmri=vy7q;G zow6`|)Ud~9J6_DRu*b<6Cpi9Dn(XtQ@1aP0Gh2JlKXU9g!(MZr!FK+qLYLa@qm2d+ z=n3!l+-#Uy%S=3a8*Wek1~S&Cv~pP)HS7w!eTgZ0;fL0d)^ z3icbLVJ+jw>|4pO%Z#AD`Y33Pl|c>mQ@x+e&JDj7d&*@s+B4#>BYcUqdNYoaEgCHL zSLK;FvXF=UVNp3S)Clz1@PZ@u;3hoOli*H{EFj76obbFw7$&Bfl0Dh~SQcJ)se?H) z)0=F!Il+tY)+1`|7v8Uv?vIlm9$rN+O-N@ul->wtRi8Zk2?ElXx zb(mq}jxVlUq`RiSK+W`R+!Dp1==+224VnTN7-P;URj~1Q8R&60W629Ows5*8IQFlg z^QQ!%J?kE>k=hCW5?bL^(kJSh_zqs&E5uLdzG6@CM@oJuLYS{GHy0hiAzTw!@s#!7 za#wKc(1SyGlibu$Vn^O8;(UAc7Ul;=!g%*B*pVs<>-j~=icc}*;LIxcXY&~Lt6w6~ z;{qgS@gOntNF!dcv*}gejga+X4Q9{wWB6a=C+f7F9}Ot43|<-Pk_VU!q5yT(5$8+Z(Dp zD~Z)prVee6k%T!Az-e#B{Tce2jD1?f9!R-}b8ly{J`Crg$LRo;T4p?kh{Zr*%6jf5 z?fXQ&*pYbeUEJN<^*aYp=f1D&^Lf9I`oQYgcvNFn;7;)Z z)D>!jeYRrU1BQ}ZivwQpJUJWd3$3BDf_;a7J%9spk2xRxSWa-rMPPP@gZ!a*vT=yr zX(=b5-S`+P`ajqq8?MpCq@o}6ST21FX3xb;a zXKCO+VeX|KJ6z-Ci)ZKG0=M%KVCHcHnzE*_{Vh?9+vkL9PG?fN^FN5Po4IkJWee3` zd>9N5B?8}-9Q1w0fg|;fC=l?OCO&&W`!?@^c}BB&x*dTe!J-y|6raFQwGDa2Hy;Dj zRKanunB(>D6~y~YV@eEzh{;nyyf=216yDkbHf%5Z-DP?1Exk*4Vo@ZtP7k0?EKgtQ zpA4o-yhd;Rlh~)V4fwqj2zlrSF@`2|^o#>BUn8$26g-Z&r zlaz@>xazSv$g%Pf&W-cs-t&D#=`FsYg+_D(a~B8y+7|1oOZL_;dA1 z_;5vtYoHQ@!&aN&jj<9+o0+1b^a*gko59F0i=g|Du)V%-3G{WEE;NZqky{xyF!FOJ zY!bRlqN|+f7tX;d=ayseRPYEm6#*Dq|Khl%W`J??6>!a+&RddfgfG1+!0~b&HS6ud z4&G}jEL6x?>b=AH6(%tHaS#3ooeHZ?wBqM;I$YI9g3wLoz~$y%a!oZI$BoKahLb1o zE&l+%0$-RR&AVv(XM~3BDTmZw$I!7ei%RH+RRwm~LvY7={A?+~EpyDqO4TW_p{oFb zHxpT{YSJ6wnBr@f3#idHORlS1?#WbY?tjH3|19V|4qW&3uEq3WOj#1E7`^~ zvrI)Q&8u>1KMH*-r*KoVWZ+SpIqSbPWt=tkVA1e)M4@nGtXP)C)U7y&$NCToMDbUz z5x!cuhCY2P4x;CxV3*Tb7+Vla|5ICr3GBT0&pHaf3>L!6#0bu|xvZ1#eF;oiFwVSR zQwk$eO3K{@wQ@QbqLN{gUWB* z;h2jLZhWzf)K2H(+9j7rk(~to{j;9>`mI9u&6ld;Jd1F*umNt08X|&61n9vjcbSpW zavI-o1sqe1;c!3#^H}`{%rCeBopFBnO+$tpW=`N+>9a)H+#BMH9?{^ag(Ty~NsM$0 zBgtpXIa_V-qXqjp@u{<5eNP=Ky<&4U(v{@WwQ0OPOPx7S)I+1O5^9=&EEqqS+TJ5ye|poJffUt zVcKW!jTYL$=xqHH(>5W-7_u%iUMrnYvWIUEQ#toeJ29Ev;kd1Th%hYw*>UqlI^C;= z_00t!uh@*l@>O%DzHo&XLs}%LE`-SrS%y5zS2Xn1C^a(WgX_Qckv6jwxE?W`_eZG; z*GfMnJg-+!qBIp;E}X}VzGPaH_Xfl3GI8Cx1-uT;NuxdTdthDWeb7-m6AgLt|&9(>&MvOiOh!x&6ir z6F$oELLA+|ZR<6V<}bl{Re|Klw|mgtE6(!?c7s*2zo6&hQeJ0e0+SF}gKb9 zsVx=0o^^(z77ub(FU}n~0RVF_s3H@**1Nf$M!ATK8}^-U_P# z%GKt1{AnZ8N`$a+d>L~1_v75EGIkHOfLC4hnmOQ?#WGuEcuIz&!1A6QSh+6SJ|^9k{qoYV`m{I2yAxejY}z~AZ!rls}z%O z;y!S^(~1}G-a!LdrnL81G-0QUWG?H0R&WZTqiu0GrTnRLvd0d^D#aXl@y;E~8p`t6iB(vaWy#%d|8yA}n^@(bw9=|)Mldf2;tH_AO1 z!lUfnBxyk;cHVqR-fHbf(WcjA{;##Lzv2|Fsm+4rf8mCg;=3|9yY7j zkrnUma$aQJMrZA9Fm0bJQNQ+`JXctXs**NP{Wl$x-3L(IW;K{S)aC6Ev03vQp6|SYo2PF^&8Hf?mmdr8Met*?WaVSjQk8$5}Aoub$aDSHv!A{vUNH@6Zp-C&9G}mD^R$Pa& zfe{*asGZnNuER5~KWWJce(JWvfU6v~l(+GOJh$Q6XZk5$i+AYUG;sc~8iPQF*Qq^) z&VO$OujF?_rSCilK2u5RQ@R0G46)rIE$+2iO&l`J;pA@lM82I70;+EUPa?-SHIc~} z5ah`Hb%IO%P;-mvSU#4WM*OKQq`E1KT{`(8uLM+?M9sAmby%GB=61RX+Gy9~n|TrVV0Q(b z9%sgN*t-hfKAM4THV?@(ssHn?Z=;yS9^SzjFHlhRI=%gK0e*6j;LZM?Ml;={x$4qm zI5J};*R$^i9Zajkmf=Q3 zFQKRNyhztWVea+iI@n)Y4T@XZiRM$l3a6WFpPl8ejd-K(u4k~sTb#G8ON3{T%RX~u z?73Bzb%Zn))4SP~VAmB4fjfBM9B2kNe6mRTmI7#ey%?r>2UDL^1M+D@G5v2vKiE%A zLF4QgRL*;4)K{UyRVuTg%VQqHSVj;!-0h{Zzb;|v3xGKjzsW{1mQm|5PPyveX7WU~~goW-Y_+C?*SL>1kn)X8##ZxxGteR%r?Jo(2a@*kFNHdms zWisX?MZ~{48A{&+*FTzd+fO9YsWQ(w%a@2?(+7%8#+q35q>E_GZzJp4`9UhN9QzK1 zlkW@HVEvv!xbaU6H{UrD!I|6eJ#94-Q0=X!Wm$Z|mvJIJ3sRp=fhj+S?ktQDROG*c7j zeBM|U?Og$PbWh`K)?MJi*A0zEBSi9c5tdxs2?}cruu;qnHttI&T^aK+dHVxKvt9!P zuJUs`raQnw{#rb@^aQR-C(8GEcgue0DVkfQ{zjjZI-Y|&j)ma2 z`4#Z1Zjc0K?!^A&eKcBilAc?z3^rFC$3*K>tfNVjriR=$I@r`wRpu6s&wp&ix4W*h zTr3`%9u?=~48nJ|o8COH27$VwRb_?Z@cHyJI9>3OW3=ED z)2#lK)6H^BW6d1lz~T*5C7u0VhC$$2HAzAVqMWA{G=_E3_1tXad!0_^EvzBdDX}10 zp@TA=4bbMw!xa#SiZ4{ix)(g6c2fdvoCuy1m4X|Edeqic7Nz!v!br0O8CF3Y`Q-!O z{X2|S@4Qa`t+A)>2W~UmxPH#5$N$K-j$&G0a)tGGE@cG-xmzs>pZ~DTLjxrYryk2rf@erm2T6#1}P;A@W}H8xV&o$Johi-#I}di zE6sC^9pl>}*rNr`P2UOHK?eZ&Dt^Mjzt~jac$#y8&m8DhC}c=fQ$dmcz0oj;5D%qgB^akj;63cB?*P zfOiHtV9Ud;4O&F|@)8Uwd_Z@6ze@|%7ZB%I9}Jy(0Pe-zW1K`9pzv50QFb@Q=XM!H zvuFy@zBU^t&Xj;yq9(eCJgS;TrI0rkjLYrD=tIFTL_c|uTD{dp^LQ6h^-r0*sqj9D zogYTl6+VH$1rl`omM5IveN#c%HwU8ro+EGDF;x;jX`Dtf9QS4C=$dupZ>9;E-5rgO=9iX~sIp9rZ++-caLW`mR|E=d1(gRJamNhJ)VP_f0yVbmjZfSxPipI zD<;2&*3g6JvtdAZH#5_d58YSBfs)*G)X{Gxud|KF2hk9=3w#}`u$g50DRS;z%7E2Q z)8Na@Q0nh=3HtZev7ZqFVL#>Z!5S^7JHL=~{e%zBaqPg-o38j+KdCb6x)(e1bi~o} zaCjzPPgj{AI6%Mvel!RM0hf(RlSFo+;=W5LCK!+n;w4SVk9xYw4?eAgMh~}fz0F#G|*_SIqVDPqeZ!$V+*nM z)jg=uA7s*^F2jRb_FgJ-8dO}YVZEF)EZGx4_Y^w9>WWg}Ta{CJLiizA*O%dfeRGiR zm<|Q=s>zizt#qHu2xJVc;}z}BgQa`Su;v2~9ip7bwQJH?ZYj+@(shJ$tt$w|R#l+f zQ3UHAUm|;@mi{{_2d{5mgqNi$xb|I6Rk*}L5aMW5)srwdG-V?sG#TR=w-?ZB9!^S{ z#CYNhi)r9HO*j^Fp3c?yL)I@^3vmUSc&fl1;tr1yEzd+WKkf~!R*37fZeU{A7+9AI zk-3X)aEEX@3XL_wREvC0rrT9i4nBl8*m=q#tBsiTa3AE!ux!wvUu55|Y*eecPFt0Q zv8v<(j_s0%2DlHE;rGcGp_%Q#w8VAdxItFTyL_*kQEFRuJ0+ZiAJ3XjN^=DIn4O=pfj{`Wd9Kdi`UV zp{`9@**S-NpE7C6Qh;qwK4S1Yw&%eS<}GJC`^z{z*REvS_u3aEKc%5{V zPRG4nIat96qw0psplvdn2wvX^_ACQrFg^^`Hg2G8&YN*CuY?|BxretD_`%G?9A4}S zr#)$D=-PY}e2XM7@TL|9Gy(njmEevuoirmj5$&y4f&FCVH}(j zSc@WT=cI@4D&4&P4f&b77dJ@q@uIzLAYY$lm?_v(va*M4mJNW98OJb7{3!BWHipd+ z-S}_b2~>RS$L!oB#2e7|#KG}cT2wIu_Z*o+EFxKlp1uIDzw|B01c`(1`P<;F;SEiP z2Ekdli_^hB#+3e72d`>hqLSGL&X6O^FBcC)o1J5vJ^%Tz4o@$9yKbCrt(C_QyVcQ5 zaS_T-Uc|j@uEIduAL}^+cx64no%c^+Ft3thZ=uZd2onL>;sA1|RS<NX_>d6qr^KK<5o)zZh=7C}{jye_ zcj!zdm^DU%ky9v4Tz%(B3hUAi6b6-%22N7r}RavQhcM#J@2zurvf^twIZK~VF=6)7}Wn>fB zwwXbmdjvI9IZYDEwb*BnonV=xJJcz zm?~4R?F1e6NuV0kif66xR55FU$+S}16q91HbW2uPj zG1dvK20A7GksV>~kaN!ywGw?{-Zm-fYS4)BA(d#Ww-}Q4gwv4~*^KBnHxAcP7hlI$ z)A|ls-n%gdmRi}7FA_62d4er8MNtO4xn7(P;!%w44;9R^6y^!uX&`m4_rQZ$>9FL7 z681~4h5vUFLN2?}@pRU))b$g#IzK@l_Ab_;6@mY4li{72xf9(>^qBeMLa<)d3dC0} z!>z7%+}Yv`GcT===#d}9SuG6aUswe0i>#2}MVsxU{Uwb2W%dle4+|Q7!E|dLel)mQ zWvW$;T%`hLd2|ex37X(oZ$7nD`aoCAVW?|b1B(8zhrAo1@Mhu&G;RBIT?f>6n}GWT_AAE zo}uUVCK1)%cG6nuiuM2ILxSNm*tB~))UZ9y+eg~5`j`mslSlzLi*=KW!S>kGGY1!4 zo56DYE|cN!vw8dG$wBe{NZ9<(o$kKPU-hT(KXM(|y~Q61{JiZpsvY3)di|5|KgDJU z<)5H)(^~1xL$0uVRx6o0I~n_X&f&qFRlxVsluFg9VP4sMJni%Y?J}#_E+dj{N-1Q^ zv?2&;m#4#QmZnO$%X?nDlPO!g3g*d<6}ftEIJ_lbDSwfsnb zJ6yt^lebavd_DT^3Yl~se zB5ShhJGUbFm(qP6wdNNV;RdM7Hc9NKTPosUjmNRN^WacxAPuxGMTN7^G2F9(xOeS^P`fGA;sKkdbV?wGA660TM^Bk*k9?dc zTubiCGUV2hYv7XmY2|&4VL%QlQUe8b&1hv)q!$@F3s@9<$v`3mtgyA#5g3 z;g1SczW9sd*;HGV$L<0g4VBSmmn~*}t0We2=D6C5A3fK%5L1DA5NQ# ze=e3_(W+z=Otpuf^?TtG+eL`nd>by;2*NYqK*rh-Eb4TCgcK!EicI0;uUks@j~c;* zXE5%gw?J3y7nXT$AxeC1Xw;kt{J)oD%8FOGdPg>#bnYWN9}DB9KO>bbg`yDpyn@Ux zxz5b6Ka4Uhp}07M?atp>LUo%w!DDJQny!C}E(c2Bj?ow;e(kWfq!6yD9L2JVJ6PU% zl64l#ahK)Vz;yKpymX)(Y=#e$h*vVyTS$!eNTZ9!2>FwVvE8uos2gTR6e9nF1St5G z3yqg0cs55XapdiL(4EV2C-f-#32rBH{2SqcNCB-8jv{9w_(1c16|?Zsc@P-8!0FtZ zMXnqU$IlbRG?Qf~R;ZOSGp1;P0oe=ZTLrjI*6ZnK#vBjVTH%acQBYo%h~Jt^jWvHJ z6Pf25w-nRfHN(9;7krr2XOFt)YO%e3==MW9PyL2%0J$t3qF$1bAEzC_erWdcMrtpQxHzO4s&nU9ANZlbm!n50odO1B<%-=w(OfF!Z_ZL{?6b5w>&bWKyCEU8chj4@VNx$Y97#c1H zb^csDvy#1gMyb&Kn{9E?@fK6tD&1_c>S5+d0QDdmD(_lXra zaN7c2jQfD1whx)r#|IK(dvW1vz}10eDsLBo z;1RF~a`cNbA&V5QqoSP*ac?{N~pB1*Y+~GWo zJHVWbd%!YH-!S=QgW!F)g(ps-J|&ns zMCH)3ff_Kgd_n!Km>P{ zsKDkz=Hc}lHq7fIZ|IqkpF~uG%}{X|!rdgpd*AFplr2kO#>QY&Kl{*lpz0jht~|`d zFAHVg9nmP3y$v1GBJi+_AMszxp#SC?&dM#9$nhJI4G4CEK1&uyeb!TkxRzB{W|og z#29(f*Grr1-Z3H17eZce3W$gRm_M<@sk;8K==)3*Z+^%69{!?Q!x%LQ5QCaTOXMk> z!fi@UX#6x3+Hx=Bl%M;F+C)23AK3=mVLvR_zl|M9^NEm6HH57JuxGo)!SYtzgYk?fHc&E%hRJ@vM<$AX?%p_oKi~>!g>~_;B{J{e(|`HJq7h3qZLp1=5OV zqGf(KlpWo~WWA3A2Txb*O$DyXtj)0RdnmX}+<}o;7asqdsZgx80&-nfV@Tv8>>b*G z*I3^5nkIhqnoa4~`x5ZO^C`_*yocUC*U$35OQAPAmvj785|h2E6V`gK#-ww}oR>2d zcpod$AYfn}=->ZDHhCtXs`xk~^)!{VtgHglfL(an=Q5p$-vdf{t+;!O0<7QNg9B|d zaE?R)m6n&{`gpHLk&iKOW^FS#7~Uf99Os~D>M}gh`+x`zdEl~Ue@Kk?$FnsHY0pRz z)T~LQjjNJj@wq>o9ZLl9i_}(l((#&<`<79`EOodcI1}fkw^n&MEnpdZC7_v^4H7p6 zx$@px+)$CDC|@WIg~>CybrNH=M{gs%?&P3t`a+m#B+OkpC`zuErof@|>p2DQ?qQCF zF!gEpf;j>cWSgc1o;-R3tIu7B{MgM*=FBj5MrV!DdaWdEd<)K&V82tDGDdowh7z4r z>Z;QOU!(7lC-$M#n$0=<99>SwJ$Rg^g&y2(hQ;7nk^z$ZtxQbeV%&M{2V@TMao_F! ziPoQ4-@uGEI+-rY3vkFJ?Wd4DF$f}y%KK>rM}vre4`ySnZtspGI%q+9v>gw zL^pqUjB^*1e!{Y1|C}y+!UI8!RH=i_j;Qn;bSKkQ^ zb6K8Fr9X`Gse*R=+VE$iAu;vS(7i%j(pbSy)gdKUmZXF!=S0U2B zqhZybajJRH9`F9R2_1H zk%M{Vhp{xwXny-+3dPO zCV?K_P*0}43??NJCD?7~19JV(@Yt+5+_Rg^iEujWBv`r$=1kY3d6Rd++moMO3C#k% zlZ_mMZ7i{*B=yCxf$b zL6!n~v}%zO1zk{jl!N@XKk2^$Cp7$at#W?b9_-P4OXLL9u(Wp;`k#wO^(z77`qDgH zYCQ;NUumM!u>`niwj0LR*)oDHKJ=%1I|&+B=AMcB4*q@@U}5_O)(yJ=)+JTp9+gHo zyQ_}*<=}w(^WFlj>VU!vb)@ZdKD=yA!DuyETo1ujCi^9MC-2wM-wg#c@%>F;ZuFz0 zdKxU5bqgMpX4AV1Ct!5wHs|U}mh<^)faEPX10N4_LFPaMsYv|E*&=xw$nrAyeBA`5 zwta)EyQ}e+?&_lck`IW-)E(%NHb!;-NOD`!bce!MbbxR*9++rxJw3#eoF z&!^Zt`NT#AeA;lA1h9S00@m>uXD`mHpGY8YicWz40UI)D-GW60G6?+{biu#a%0=Ds zILse_FW5ZI9CaS7U1>$N+;y?(%0*ap@)bD5Y$rAB4(LvqChHV>1oDDcKrMlFBVL+| zAAQ&HBw3eqmz@B7vv>_Z#;0+^=5ooV;nQ$pQ8>&`w?ns-zck&$2JXL|%j+=-gB6=k z!sROgpqN;VpMBXr%!)}^JuHfvpT)5K#WmCl`ib4aim>IxXHYf$%DMU08tXi7K>oXp zP#tlE!)G{^`$EYSKVED?YsD)Ny;}_jhNnYr{AD8P$g&JopVN_BY(FfIg7i59>bzDT z4iwAK*YRKQY+e8jAM?QvPsF+YBb(5UD6k%+6cP}$0dE{ZT(_eaqSCraYjhetZzTv? zf2)aIaUFOZY-SqXH$hs}UvhV%0lQwrq2}3pFil`LT4^W4pUh;4lDmS{UaY_LV=v|-*_rH}B&y*@-**HfqHP40TT?gpsi4|ZMpa(Lx_h``r zBdT#w4jS8fASsX1oJ%>Jw+|PBb9o5}b2mZUhDy4=Knj_K8E|JvjU-Rw=atO4M<$o~ z5}_Oo?)a%UFdFv?taf=4)5P}}u5}DX^J_VW&yR8 zS2YMN$3D`POQzBP9>1cCzp+j4Oh9b^7-?&HP0|uiRGE~eF-6%A zz*(gkqb@xoT=NoO_kvV#sw?pk$bg?SS_svNjnVd&F0nmn-(SNblvg`QX)k0Q&rkm%WcB=UGAB&^YZ zuMgkQ_mgu_jJ-!>HLA0`u?g1C=*@V{(t@=tqfaJ*LHSkl0P7-Y;bxYRx7?RV7X*`i zR0>oZ$|2>63%>DeVlzM6xEfEVLhZ{d%p@M?hcmk@wdWLBZ zcZj&`Y>;^$$n5^U4gXH-BB!Qw!K+^j==^9AT_N*;X#E0=!ZBz!f1y?agq$x6c? zb6ar9rqlRGV?B-Mn+p#8cZo8~gfRHe2)7GVlGwo;_-yG?Ec4k*x2$?ZH``qXgX7tx z#?uVn&3g=^>zBaoB4?CwdCoboWqQ@)rwY6Z=Ss41uQc3aiKtSuPB7)ZS+wuNBix^* zLZ!Yc;HAnwa2>gZoz7cu^Ak_{c>ijM+S5JvZuRLU*2ndBzqa-KgD9rT_0LzG?h0cIJ9bR-*uLGpMj+Zyy>2s zM=;Xm1vz?J8S)mfOtU95xaLU>IOVeuW+`OBvfl^cTbCma9J~*%?U~Rpa|75&TUJdm zy-Litv)=QdYSPNit$W7mu*KCEN00ZgTwHZ-cr9yy;-r%eW+I#v*Q02(`zcW?V({yP z0v55ZowYWjP%LW1jSKq-xvTQw?Fwf)INOpj3|4{PS2<|2>;cAJ)InwKUy$S=OjjD^ zVa#a-I8b;O|7~Zey7z7TQyM{5$<4xB4Pr$5*%feS9a_^{Qkl0VMv&j1jOPbrP(qA>OBzRs%Y@D?g%;V5tCj{Tfj|-7J7rN zwn@l$LWh34qDaOHs?g0`hBM_CKhM-W7DE&ac>;)gJg0T1zABH@9hRypldDC;A;NGw{ z4B05gEw_>6iAm`i(z%6}} z1Zw4nVJ)h%{^xX%$dh3IUdof}{sjV)?>Hl$Z{fjh*)%qh%QXzYh9y+ z&mw;*iY@+0k5(+=>AqWsgEosmW5axI`JzI|e67g+dE_fQ_s}NE1*Y6n?QF05p)V}k z#5#D=r*Jdu!zk(dKs!v*&^qiO*L7VE+5DXC=8Slu!eBS{>H=>?#suh|?SVga4lr-~ zZ7S`21a{SxSOt!xgBTRoO3xQJ)3-vqfSE1{eREQ2k4rnPv$ z{3!|s%s@mGIG9Olb5Kb%+)D%fhjSrM0@lTO`apbJ?Zfk zmzu3%WXCJ;ar7tj@l7Nv1vqG6mWE4=3W?eKS`OcU2=`3*5Ex#51B%I z+zb?gx_8mk>XjKBeW#0C9#;`ZBTG9B}Hk?}#4RyLN*_{g?(HmzoDzAz#eu-|?z6DY+CX<9p zd9OkDRxwtTM-kuIpFrlL7Zr~fe}@Wh zET$1pEO)}*xK?aPSENy`_i6j8{g7i=fpEbNJY&y-U{52h+bV;is~6+G*YObE@)-la zNW;oUBj9cJ80v=?5#ckI7+a@Nxd+rd@>{n}1i$Dh|Ystp`ASMg&}9WNAhuyCd}454V0-<2iR> zI@RtIy<^!$8{OVR`>tTzc>f!GE=$7bU~yE_$cFc!rtrZ?7S8bRfY^LrcpFvCQ4@B9 z6rXzB)0hYDvcaI|l|g-F&2TCEEP4%I#Qx*;V83~k8Z?+v!~ga|p;IhcXi39|Z>g~P zS`GPRHwq);Cb-~TGO_LHg~fbvEN~=;KFrC)ra%7BT(S~duIYfQ{yX}teYmnE+>wk7 zHgNW4r=rRCY|LoxL}Qy`x?{dSy0@p2cJ?>r(iC-iu3eSUaFpa#JZCdge{{+42OY9u zodL7x-dvjRz|WQB6QF-*3UW`+_kvJAAGo*H2jul6;L^Td1fSRuwf1g!7{7zwnjE8Q z0jjt<_axLF&%-}dcKPB z@wwnE$ldJ%&kqJsx>XH>TlA=?(?jz8b2y}gD`S6%CZ*bm^y`mWI9$39cct^FmrWuQ z87c=uGvk1o-l4v}38>e36PAYc(PQ~6FZV|Tt`PMHoA!+)r|mwyzajzWj!I*K&nBEA z*+@+`h0{Nw{^*k@1p_HD5HlH1w1s=%sdgy(hOB|UEk4Ne-2QO?ZA=^^hNUr2}A}t<dfA#8oueDNWXn+XFK%2m;!iNzL9Na&;seyR5S5Wi4BFc2Gf1_g?|+ zRht<5Sk`Iilu_Bb*cSfVUkAy{-cZ@^VVHf6hxYkbu|mL_h|Mu%WLU2F_}moYH@kwl z=++0#`sMI(*pr?eQ>MS=R=}UzyW!ZgJuu0yj2&k*FhxxdUD8VE^5I8>qq+t@?~A52 z2PE)E{q~B}F1~o`p))*JiGvkiW#NU{9C-EkH#p3Bglm@`!|QgDP?Nb8oj5GVZ4T=R z)$H9;L98w8`>W8_p-XNB5C03n?PY<5_Un9TV>c`?iA z6AM1Dtj(iF_wK;M2fN7mm0y^O0ZUrC6Y+&<%Q<7lD`$Lems}z2Uu`}UjJ1p5$1F^wRpto^7S-jv494?v)JK4GE zS=$TDXggl7Pj$9hM!2-#+Foy8jIo8Wl!1P!F4h^Q^soWUmb=g~b$nXx1 zEY^mulb>+cHv>|Au@r{gPQZ|QI`w!J1+#3X@Ya}qB0F186OmhjT&E^wlAnGStyNsm zUDOYJr1*IO10KZA`U5=~!1fPzo<*&dE6{|GKn^JZg@XAQG!lk88nnP}nGF# ze4g{1`+mP)ugwa`^59 z@XFuBlF5N4khkm;5s^L03;xhTMP?nv{i7yme#M;V9~rK1r%ir2w-Be1S~>{+ zBy+1Yo3}WF7aG_~iToyN;-F6)jJf-Hp$;^@4gmiC5&GS_6|So_Q(2`1sL#I!3untv zgC*^i)Oj)%NLRqn?OG&lxgg510-s!ak0S&2u&mJp#BMfnJW_GitMC#i>J{PUhF&sj zu#03~z5z5a66+P?VV>S{2>Dh_j~`JcrxwXG&x1uUUrzm} zuzDZ&Lw;R9Rd}-!e*c|M)rCvOH2{`orEzlbd>5R+yIs=@NHmAbHZmM|#Z&V`0(E!N>P=LXvP z3tt|yf#&eNV5t#LeG=-3V?`7i>)l4bvm&gv!x3n}!=ZTe79#?6;hx?$*#H?F*Bjp8D5kT26lQkp~QzDHBong}RL@iV#_UK8S{De{ zc0B;Y8ZYQA_y^P0OR?d*<8aT+D}*MvLyw#^&TH#}!!P1s@k{~s zU~wLpG|$Da+qvv{gaVEQJ%ZLnJ}|tX7-fX}xu0bXzFwh^5&DwM_kGU_zgM|q2h z@?(GAP+=j?5Z?tC?k_{1eRkNP8wL}X2!n5aHvdfX7}eNVfU7{nn=7 zi-pKPlBY&D&HsuKO_roPhVzbWt;f&mktiE?m9~sbgEGe+l4ve~H`u(&V`n#@_+w$@ z9dDsl8&hzy`(3JBH=X^NEW-+&x{5a)lTeuS;b)tCUX-#JD>`EtomNppK9tYE;~bx~ zyVekfr4Q0|=P$tJ?`yE4rdiDq0Ib08^d7nt^aW9(A&E!L$df=xh0gxEB#Vh8g@JFl;{&&cfs{VAr<)+){q?wcW zm-bX*WU(Xa&Jcy1locqJ)x|S<^bY*)SrV7QU2wJ9k}Ou(LEjHVL*4lWxNwsoPQ1{9 zosDuZEdPv_%x|Y(1qjcyo5g9ef~>>I3vjQ^hk85h!kgN9Sl}(pri-nB2aDTyhIi#r zG%p!mus7jQdELfO4U9Joq=7Z?dNtn9Yy* zd(1Dugwa?cx-FAjJO7v7RaBrtc^5$bWj=|$cZyiI>!XW+BztVE67HMiVDi3OG_g2? zI;S|(nL842g;Eb$-jfcW&v){Vo228*<7MEv!h*W|w~cczwUX%aBj6REhKmv&Lg3&O zMzmjs{d>y^FM6lr&12l2f2T5r2m0bXt&{w-FSv-eVGOykdMyzPOyOB3 z8__j(7twL=CQz8P49+c!gRUPpv8nwp&RCmGPaIEx*y@$2TJedeC{auM_mz>LU~5n) z5NEBgh23$V|dJX z53j2xA7*9bQavsg+7LSjpK2XL<%SNhop^)adomr&zgF>PMX8Zb2i!@C%PLSRx=U2+ zUqVghYG|2ppEl0>0W04uA-mTKpvrwWn4@!v_7(fW$2ddmYR%zx_rD3F-iuqf=lF{M zdXV=YBBcfeoWItR;|BO(%#kacFYXS^;qAaCrwsmV2W4JxL=F))m`Vpeb3L=I!SI88 z#9HHEIM;9nyku8l&OsK^;u-u?{g$^|sfo6CR)XD_H!6R8N`~~J=(h0%P+0MYe`!rW zEUR&XVM8qv(M@5fWg0s=tpytk4&vVj%h|7Gp|E`NJZN@#K~-k)>H1a%4!kXe`*la) z$4hS_A{PW@w+`a4MLs4hs=k)(}H~7-) z7sbKs!VU;-mLV5x4nUCqL9kqtPj+c)(2R%z{;9tWSXSsmO8jnv&D#Kcw%r6rR&Wl1 ztwvbsmO!;tbKr(S8MO52K$h-7Ol#r%a2_m;Tr@@Q^CU zsd+U_%Z{cJpD0P-U)4p&Gl?|vH-&K0~tgC*jNOw+dkB7w0^jzK+YoNwH(xj?y7;BJ1-=18m}gV2;CdEGTH=ZP}m0Gjnr* zEh8$-Zqo#M;K5UJ>aiidDzJdDvyaJ(lZEj9PB(e9PLfR)l1AG@m!U9P8wK|zK;3}~ zn6ryxCYHo-`5~za^H;a2m46HUsc#5ghx%~iPrz?ddi?2J@APS&I=j2I5jvFB@%Q~? zoa!(g<``O{&)f4T`T9Qi-YLPGe~gfMbQQ9*zERIWc}8@<55F_-D}Dby3+5*1LyfaO zO?g&IPky^KrRNwFep!X47Pe44RE?2yJn@|0ax!`;hlU%7La4VIWIf`XYP!L2 zRzaL{=XxJb?ek(h7r|WgWyA1DIJ!OiMnXofz$tXV zqSk79*U=BQhE@#T0nw-Ocq1;LfK!FcmbBUu%`m83-1lBuH#Ot8Nw)}PMf zzc}{;N(>T>2K=5<>83e6<&E#)4!sDau6M|kFCRdB@Ht8O!oarK0T^m}mGq?UAvt@? z;Pk0`L~-T>=Im2JIA`xgy4&mE(RTsj9O;buyEoIm^;M9*#*L`WNyVT830V03eubay z22fix3Dw`Jw z!QW}duzq9^LVs(4`@j8ck`?FO7O4TznUm1JrVl!b^zrs_JM!l%=Yl=wgKB3T!M5ip zNUjKj50%c?9u`T;D_ybX^hRnH&=0~Ydf+bWg(-&>*`?Q-;J|~U5N!3Fgsf~q^+gDi zMiTKEZv$P&%fVE69z^^W|h{zUK@VfB8l4?JfASPCuXdE8mVTb!~T)HYp)mPt~q%UVZB46cJ`K3}Ncg*MP- z#2Lj?ZqUMxf@7gEt`sWA#mlns-ottDJo5@!D=&@5FN!kDCVr(?TH@&jr$5wy=ZzUt zBd~Yo7Aj_C4XzV1;L(64Jc@M(&zMEn@XQ1X7k(kS@*c!PQiMs7(KpIX7sDWt9oTQH zOseh<@a*p|q84lCQAf>(hIb-PLAk^YWWqa0MOY8IE5#6n%u$-M^ASyy7^X$szIox# zHhM=57>))+n(n=azZH@YRvyA?=tqO~r7j~|sfGBc_5)8Mc_BZb_BPjjbOC@>+a6wb!B{Q00NTa-VN@l7j<5Ow z-S0}^-VR}C*9_%P7g}u;`+gr3e7yzfw%hPv>tlSOxdGn&Xoe-*1=xK*wnEf{HrOSi z3Ob%2AtX(Ku`B;Ztxjc-pSwAi-U1V}T(BE^e#BM=E&2%+s;Xf6L4oOve2gYVHh82b z2cGRvEBmRGos{~*_%0M+N>!MIOQ#_6&a>A7Jf+1gmB+0sW9|pGWI8= zQo$2x*l#uiuHp+as+2^oJ`aG)i9sml6Gr`-e~@2al5kJ+aqw>5PYo>UL4f^EN~XLg zv)sSKrJF+dykHV@x^5eDpl=1FrE5c3k1o5hZvcjqGU2FE2o`3fp?icYF1#ns>Wf|> ze|7BeX!%y&VcuS%Y9fHP2L*T;G>P}digQDWt%2fqF?7>I?!DLPO=@3gvx!PqV8&h@ z#^ALAt8H6NGJkKxy~oeOGI>iv#uV8hE3RF=E)mD(gh2aB4|upqjNNW>7Nz zCZUZ5xNhSuC_TB6>*Q77>eMkBeWj8fxB5vOUNnN*9yvVXAvtFU9$0Q6TM9g|-)pi2u>0sH>NS<<&P} z_rmk=Z~jgU56C88V=uzr*yMBic-6fUq(>LS z>aq`{<@#KP-+mV?1vMa^&4R$S<&`Vp*VCO4Yrpu}vGM*~4OCmne%hFFteq}bz`O2L!+Z3@~@jgBD z$k*umja>K~KSVD+S0ssMg>>aMDYkRmjBn9yjK4)=z+Gz)cHK7TyvuetI9(DJsFmWW z%^Ts1?GgGgIuAY?$&k5NG$Rbj_lvu5QHUV0V{Vrk_GX+s&wdD5(*v=1@ocu;SP93>q}d4WTzaR;bvvt7 zS<6mtr@69=Z)nf)V;aiH&zepueD)Vr<`EsV$K^s>pscrUP^A}U56PTPvTBV8|cpSKq;$@VBsH0|Hxf~_!kD4 zd3YT*YVU$TZ*|E26+~Sp#ll$50+c&0$oNf2yH)UJuMoTRU>2=4KY{LoefZceiz<5s@(cSD`QPr0!iH0YC9!$pJ@ng^#(hnP?48Wet37MW2BsYx4(B!)mQ*=Qa?iI@6$bZAIQ`rJv7R1vt zjz1vr$ZBlvFaeQoswnfpAMIZ}hvVfrxP4PDk?xtwPP36?Hpqq1kFO4+lNdMqa#ANk zzT0rmiocc5e^6+XRv;7WjvM+_Hn;Zs6s+0tn=Vk8)vcKCzbv&lS@ zPuJ+DW=-sSqQ&1g^q1?7?c%!Xek3(kn?4e~jgMUJWAcfKtV@)Cg}BW>SZ6ZJ=$xoE zzHFHaJ!=9{a@QQxJUtgTo-c>KJ}qFI=QG>&oWNp973!Le&?P7J;muT2l5t)YXB*^~ zO^<#_27bFiw=l;q8M{dCN1OwR6Ghb7?>K(@l0*NU3xoh2J5ccwp+2b1{I#!xYYytf z*y<1e>Equ#^MB&(!hQq7ACm($$rP;E7l(&(91#=DU|XC4STt%do&}U#)a%A?XAZ*i zUq&4(x~+N65Xa%0Zlojpxq@0Uwd|g z(a{qyb5aP$+!kY+ND$VzIFXpA>tXmtJo<`UB-SUpK#kVYKU`1YAoZ%8#c?d}KF4uU0oo58Orb!;e5I zK8*@YIuFA^dGuuHCiJx4O*IU9>2$fkidF5uU}t>@OyK4#iG4{Vy6QgeH~op%b!ReX zWW!-;Mk?&Q2k?2~Wn_%^!OE^eh!Kf_j3!-}Y@o&7IVz7YC;uP=3$h^dWf$EtQHT-b z@|xY(gUQT-nK-j13aeYnQKU5u!bTs$RmXI^Rw@d!+YZ73(S?lj-+i#zQk__Bks~s% zRoHrM&B|=uy|_=RpU>a`f3aRAPLEeXzcgcd-k}0=e@}qCbzdrzA2z_J`Z#>}XBk-T zm%-BTm1NKDasIoRj?}0*8{7oy@ouOej?5CqKrW}Yd0r$uYY)e#>=O`t{}ek28S;8X7C``w!)Fbd^XA&y!Jr5R%3Gmo8 zfZDHqMNca8;UgmtY4RD+>bxJ@MZ&O4{~LKTa|beK%kk5C3D~S61h0NHpn8gT<+p+a z>}1DbBqxf7NKJsb*XNMN!`aw$WI8-nTuuKI%b^V_Kgg%Jt>9#RfxqJPO}-{A2LmUciY@pA271*y=uS21w0_47yWuwl$K_~eVNRt}|_t53QZhFVN zoVT0nNqxnzS%u{BeKYuy-9v1MgrQn>DSc)p30qR~ASJsLj%}SlPwhVdhbMbrmHrGi zF)55p`(=U>&N^6@9gYU?kD~1GeOe_nNTXd=__5vTLY)JZ$M|x@8s&nxhei9v8^8=bknZ!5r&QNQVzQ?&yHBS; zs&^nJ)Y$Nfzog=wR9!xL$g&Z!()w_ypNUpd(wH`==!$a~AwO|+L zD%%Bvt;cZJ`e>MBu@?63sKpQcy~O>_J7}G#jBtEy(3XaRR*-y@qy+a{UDz8TPHaE^ar?#p8C#IK}2SQOrCH zW?XmA#daZk&GrE>MaCRIJrqTb9ER8N>!3wp0tn8|0aczeNcLVvJ1)=GZ9M~PmFiL4 zwGX+D2R>6vfdk2stjF$j-agJnD)efAR%LT%`>#SWG5QNuGK!!Xnrfubb%FlfWrok# z-~6U=HNMXGVS0UAE6Vu$<0Nix{Ob2GiYq11txOi4DE*6ZeQ`K`b|yO<{S=Nw&0016jPgYjt+?9Wgo=7@?RV`e{v^`H8j9?qpPNCpoO(Ebw~|Bhl~5kUcGF*`c<4fdV2#@SDmAtb%SXAYB4kO=@rz7 zn+p+|+&o3KNEt}zeqdem~-iVjptz(vq3Nlyk zl;RF;5hz~O3NIh2vb1vxqhR_2UHZS`*!;Cn_Sv6Y9(;>RV`GGkO@s)!I;e4NI2BxFQXfm zL}0eZEAnpsM#yu&OQ=mIh~$;v-0^t)FDDl|&)mYQ7%%L&mw*QhJK^?Qd2}>OL1z~( zTlQxjyEZ_Y-M7zyJv-`&E9|t{f)#t{{m5HbeOVm+%9kT}e67-BWq173~^{((`PkZnz z!X%lNC-(56u?XTLgqQ`7y6Nw`mc+oq9Qm&W@MhdI%uNv`4*feJsQoy}n3@dRLW<#T zyEnQFh0#@~0_Z!vIy}rVG+Od)i0`8Y&~@OR>0ED|aZ8Emcybg4E)-(q?`UjmOXYTm zvA95~6F)w4gTh<67;}0M-|_^6M$tV-*Hc3)Apax<4k?>;cKVF5K!U z$J|@d%A0-YA0Cvq0egFGY|98TIy6Y(xN|HX%M^gVLn&O}{T%vAbsLEvC_&Nvv$?zS zFnvGfjDKf6fk|=aQEoaA3^<5q&;OyJdaCSJ&s@Gi6CW>19VHXXoJrbA6?!|$7)7+j zF@LrV9(R$XBJ;|qtLh(OSpTvzBZQ%LItKja>N@Og?dSZ5Z(&DxDtw037|2heqxUB> zk#pM7<)0X{^ZRA`omOJnZa)+k6hgs{EZp$63U+&2K~Z`YTD#_vweP!#o}VH3XnZH9 zy9@a8E&FNN`I}T1*&!E$0&bvNYhW{m*FW{sy`_@;v?JDNgP@4yNml)FIor1wsVGNXLzZsNTut z5h}Oi-c`rZ`r{>yKke^gKYG?S^+IFgXEl>9o}AIh)RVDXmsQOeR~Lvv>XD# zzWF-dHN6j&^_QS!#}zED{SQsH9>Vz8PI7!-BVFfZN3MzA!Hrhj|Hl(G)Mrm(WCu4d z&}h_)-K%tsf(-d#Y>zj-n88c#d&)4%mTsE&6#PTp!t#h8Wy56y1WIS-zxslQp_;=GeJ{uc1Up05iE<7fuN_8u%87PM=?rK-V%TNq$E&)Jw_C zg&%08y*2KA_>aDR!ovx-$|`Es3xLSK%W!%^EK(T;$L0pYOq1=f>8uMpUSEt)eKn!a zdWasHp+Xu~Jfu&*uq5fbsXugw;0-j-AEV=f)4=Y? z1GK3Np`))oc>N2Cq5RiY($TmIjnAJY_I2^ld-x*M{=9?>(@V(mlnpSUKgm#_K8|PL zYewWZSHXlneLgcy7L@wExE)0(xg8Qkj#@jy-c`0#_~{&|I=mPibVd2aCX4BCrwuHV zd&U!155TXxqj8JKb<|pxjC~vCVy(qK@TPB|{cjokl=bKG{Rl6otVXu84?OQylgs91 zJPywfr_Xee(NCQBD9D5=v=xBDt7KHXdXjT5Ou)bAzC&b#4u0=FOPI%rKuvGbtj2yw zRrBOUf13lnXVRfVx|1#&nadNH*+Jr~xcLq1ftn`s$Vl0Ccx;mfGlK#NEa1-E{dsh= zT|JDSn#R|VEhqi2^C0;7A?l`Og;#5}z(ZC5H+MOc78y6#`OycB6ra&;o;onUKO1{8 z)5w|mr-)rvd zaj`V(BBFw0)+3nxI|U_v3V~IT4qD2kfDqS{I}!aC28K15_-#VWikm&C7UYI}_nA6dc&N%cxVRr*L{cacT4cJW+iSJWWm>*rH{w&;k$daFe(!arcntP-TH)-ci7NZ z3YW?BqNhY_l=I-cTLPQM!>E~%Dy(m|0MX?8cri zCz(y7{s>@4TN>^M3dVOe;ovEAe!Y|^tFg`>7sS=#g)?cO`1OTRQTQhsFz9S_=t&>A zNY{X?>3%$%S;!BzIR#6VPjH;-8N6*yf&6c?*P@(wCVsAUhba#q;6=M9bZX`CS}$1b69pfb)5Rl-LP3~@*)oHjQ&)75uYqpeCl)m7!RND*bE zJ}wZ>I8n6TmjU{D|G}$-i6{|v8`lqb=&I}kH}1XvVL!vM)IV@e&NN6h z&4&js1hMODFr1#^4^11LNX;KNNS$p8cA;nK$cQSQzT{)1*kF!zLj$N&sLeFz?&bem zn+*dTKRicxgjU1_5PR*qtv|urF;zSII&U*|UGJC+^>Nnl8e=l6- zG86vyB-z7t7hvq?Q%ry6#7q1xLV`8oNT>A~oSJD1i}tqhc34^B@e`Tgrgfftu*YfVv z4yxg{<&g7Tcc`1EV`b^J!@Q2GE%e_#71lp75>CuBg`D9a z)IKoAn-iqYtKkAJx05LpOiiGIxlbzBALvHY0%2A`G?H(5Is(T81i{DN4%PN%(Uu?n z)bL#}FHM`7>>8{bCoFsMu$LGydBNPXsI^51)V zOl`J>hTHaJ`p{S2uj}=6Uc@l14(#IBF7d(Q(cR>2b}2bIYdHqaaG?30J2BQa9A;O9 zz%{XMeAn!Q%Z^x5R!an;I*!6+_id!XxC)aVnZk27LA>dtgu3>zDAKkLTo35uC8I7h z{zyq?;0lPp{vY;jALH3`{P>MOHi7+^8{WUO77Vvp(7DeIAR*0{#HDMJ^+lKH`sB6v z$N3sL|D&xkWirM0R=&L9LlJb#cfd7YP2hRsJgl~TL&SGw;CRUj>OUv}MemD2ue2I9 zSBSyKl*PcyItG{h8_4|MH&C=)3+wK`A$x8v!dmeTtgJ`0yz!ZC9*!fI!dK9&;(4T{ z>kBTH$%LHd&9HE#I$kz1rIAlWAfRY9=zdb7%gh7taPbDzuaV;0h;GE3G97-i!#)1W zkQa376h5R$aBN?{Jvgk_L3jIxLH&;Hptt=R|E!cNOsjrP%e~wxKNn=f`}uG2(x^G? zZC2pTviX1oPo#MLmHNo68{xN?g_5s5=5Xe>D2%?-2A%jM?#|%=f~Puo=N@lk_@Ab* z*TwtEx{gp9d1({}dL^l%O=r3CLwmg5u^AS-ii79HyI6K41Fnl)fWK=4Anca}KH}-~ z^}c#>Ox+Ov*hE2W->-m;GlN0RTY{06lH)H4Fv3lNrTj^ulElm1821!*U|(lF9(*E$ zZUSF@l>9{z-17PK2O6*U)AEPx3MN52o~b;FNw}c)nQ} zE?DY=R@MPf(7J$LiNjD3i@KIB7No~Lt6 z(^gnsFo*CbEg^7j3)lyRz?Rk^eu~&byuafmp7CZO=I#{sOXW{$`PUd7ee&tEjOmyy zc@_M0!||b{B5dZ^b9KKX@IL1R`99tb@zXcJb^Ei}#O<-3YrF-i(q=rEyqu;e|Af9? z7O!ZQknP-AM+Wj~Lg*u4mX?E^i8L(CRUt}qc(8japDxX7DBJkfaRnB*2 zi-C3biAY#Fe{Vt+RNU^twh$?{_@)wQ&WIqr$&1kO+-%P6HXpQBA0oO}57NcAvr*7f z4j2BYM6JW#@b>g7-0vDhKP-;MpW?x2S6N7H7Ko#u+!eBo^I3+dkC158#{ipXJW{@t zzvb*@sFKjc{b%@4`X?9uURg&+IhLXLx%qH_yEEK7w4Q`=`^dM2E}(9Cm>zGt#1}61 zgoJa?sr>$hpbjg+i_6({^af+ym&sV#wi#VggE3a59BZC8@b+aCV%P88SYIc|dj5@r z0T&(E7Rz!notO%1CwjGyc1)7qm zrb{pesv5Mj+z3u7EMyBlE3ost&ePt!2^e*_4c7Ef?rydjV)~Ym&;FG>g&yRsIV}r^ zW>^EGX$A3_Z}4kk3>CUD4KB?%iWisEP`@-+{L$o0233Aiv&EMDbMZx>w&4=Lx2p#& z6<@$U(YN@#xC#?aNmHx;ZV*KUYmoE2&7aW!8aGYqqM5+JUbS#q`FI~BnpKgS7su#n ziO)E#q5yZi>?3zJxx=o~CNysKg-cY3DC=49cHdgYs#YYxjZb#C>9r|}?^@13a$z}6 zuJt2f#{e!nCR0ll&NsKP5Y*Cl!=DWZ*~9iYHS8KsXtO5M@IwkU#4G618-Muex0j;* zx_DY1vleO!8qkp@5s$(p*y%qN?bM60Xh|qfWRU`Ptl5Rpp^NdHiUuw^;6yjwuZKeu zzoE%-;Ml}fL~VO8FQHL_y!<52InOfTTg)i!fAfpq;Q5VSyAsGN&E(huKIPQ3fn) z1=|zIy`)vRJKc<&_`4X|4KLv2S$qss?}UPyX!1`9;nnn8q<3{C{|eW4m%SE83qChf z=1DMp8&rYSpFhLCV?U_F_9|RiH6PpqCqSX|G0`Qw=ZjmZJZ8f9TQmAYOK_!Rd(*ZW$9|MMNs;krxf~Us~qaB}{u&Z=8EY*m{@Etvn8MYrfy@gnZH5K&r$`c^NkAZE0A$V+x09I5V z!>qbml8|~Cu5t`<>(Macq}u`>4;H~EV>y^R=EPgs`j|~$R52jq8(?-P1pMTB`lq9Dk0F=$dm{|>owXO>$UzI}xLt!cIL{1Y#IgLh-=u5fZsHQpN?hh}&h7W{&Gk&BrBQjYPN(}G?Dwy5yw3o0D9=f2Gz64^DMjB@{cr@b27 zyK?9}*{!^ft#%b>eE<7MOD-&X-2PL8*QYeB>>|6y$WO%qp{x9^d?V@ykCj8j= z8-Go*B5RZVF-H46b#scPen*3GTswePFI*0qno{)4?J780m`OKFZ$fM1nebnzI*rmP zrd`tRAT078xBltle{zX{wAoL{lBDw>zjHAhV&%~}MUy|AA;VNH?IB}}cc9IqKG5@N z;w9t{k?n)FAof)osO)-}*2wWw>X+bjZf_l{Ex_LI5a+UM9A`dxvo{|5y&m_jv4llQm&p1RKcREc6TbJNhq&|8QZ{_~9TdC3Qbz19 zygfVs7N>K`oUcwW?T0?On^Xqhl4DS0T`o~A2<55YRe^?AQAWo%ze33yS1}-#fzhs8 zH2P{kbutbGfg@&keE2z?rQ}Pb7TkvUmD=oyqg~uyB>=C)-^DGzx6|?~$<(c)fovxx z&~va7OZv9Jxt3L=Rd@^A+9t`=7kwjZhcDp)o_}RR+#Pgz8U~K1I6hnPXKK3749p7C zp;)1vbhST3x4i?juSyr=T`I7TJIhbcJC8o0g|PJ*mxbG!hNCIEWM;?)B5Ab=c$Fg% z;vI_XoLuo$+iVo^3Wtfa@9@p`_`r{%C|d3k3B^g~)KbD3uEdQ{G4aRn+Tt&js>xg3mdK(({1l5@Gzy5;52ktBikcCBymWAHnEhH2k~p6MAD0kYrUGTzgEJy)=-8 z@uRW)uDhSW_!;NMe5p$B-H3u&50|r};rXPv`#;>&_z1tQ;zQ$mMTT@e=NvCpcrhS^ zY8=9R&Btq+CP%1jJRs=&;A2XvP=!dTZ46dpVe&N7PZmLyXQ5#g8^pB_@z zMHk73Fv8qUE+I>|a~%7m2#8U>$j)v30y(;p_*kbCf#X`aiQOkUF%!XYTPA%pSU|(y zTchI2-Ed1b7Uq2XK^Vy!5LUzGXr`L;7Trw*r61AQd*~(6T3m!1e@ukeZX!%okq9jD z9;U9%(Xd@xnO&&)1y&x2;XF`>cs-@1nCL;_ZeRpFvLC0Sle)>BrveZ$R}Icsm%vDi zF!N_F54?=dThS9?!F zsGBcloBH82&u%(rc$hx35QT*zF*qsU9w-eMlY;)EC>Ib73OYZjo$FrIb$Ebm@(@%f zRf4Se0@!EZ3m$FStbQRk>zaBIJoZ%bH1af{-#!ElmM$e{v~G~Ab7YxQ8*;E}mk7+- z-v-C_M#5K#-Hc(DG;^2ZS8Isef~y;qu|RN;?#R`Gq18bgS2Gz(r-e`mU(cMe7{FPb z_pnT>0S`EvW9>mMdpP);zkam>YgU>@OY$c{<-bYnjrL2tT>qK80rO47W`Lmh9wp{s z$#jO`peN!MZn}gvr9LL08RS;=TrPKF3P%SPGG{is5vSnql}w8dXf%${ zqw8gub2H??D_D#P{T_w`DRY=mGaaVH_95n$-KKv|{6}ld5{mr1Rdkd}p8t$MLZ$mnCbY2k6WmY;;a?J)&Q4X- zggw3Sc=nGwjBJ?4rgW7lLaDIzUzrRj-wA>1Z#3e!UV(9MT=sOnyTap!!% zWn~7=U+{|*ZRZ%3N%`>4gikf!43h(AtI5uyE!eYb25+}k6#v1v4x^Z%$(Nn~jOQlM z0Dr|k^50~g#o_WBoNMSXm3;n!%66-8%%sIIDs&B=dVb&qa2>VUy(9Ermn5$L^qoA? zap%v8T8e|5&tutwYoKsWn!Iz@#HoH#u)Omdo{{Z@H&xj+1A+ULFS@s4$tf z$4S{uZlC$I7}o6JqsodlxUY8weeS=(sJ1G&bx;$p-c)C<9S~u~!zVIghnB(F4ILy3$nuaeGWinmQ!l`Z_Ecu#`r(DbUPxek@ZWfps@%8WGxu@TG zX2vV{y;o$QB=;k+y>}mXgdPUNrn_|W=sc8Z`_35L>b)Iv6zwg%#wzsaN;-@^RUD`{y^ePO}7G!|Z zfp>T;$rm(E5n>i%k7jGHVdv2$xTVG!4!5}DZQUJE8Z1e-|23pDWNN`bJ_WAbPGTDF z`NPfc%>s*0OTf(Vlt9MZ2M_j+LZwCmpYQok;G<T4W?|CXsHBG>JyDW5WETv5I19Fphjl@5dCZ<7@&YVc0 z)ocl=oNxhDzVF6byO)46e_&~_CWtzzvNfyC=-6>_m?aZT&$^~#*o`1qI_@v2<~vBU z_{Z>cnsAJ(z|=X%KbO__s0TnJ#_+m`AF>Qu);fS_XPJfR4|A# zf}QPpyq~0>Y9Ci-OUXmbF1EQMSl1x~-%sS@A(e}yaK9DxnCeX}Tb|L#uf`~K z+?exe%|z*wv7l&hOrRb-gEHr*u;yXm;BZuj`?^etWyi~d(TEc4686VIH7jyWOBau} z57N_BzXUPBXCYn6n5<{xSiLVHoNd-_T;!7qi4teYQIQ(D)a^Ja4C`jfC$m_uC5med zYs=KseuIV1b*PlhP zDS+!Ip+`{$RKCh^Gx*F)Oj!ZW>^lQ>_b%eI>$0qus}k&8TFmE@V z=VU|mj~>J*yYKX*%PmHsE*RfW&xPw6Qrz7317Kot4o$XdgXQNe;+%E~FKFx~+3BL( zex*xjvs;B3k-5r$|K!NGO_y=l*#+i4JqHrs4B^KkJ*e|2CyvgW1+gpUQ8V(IIp6*Y z(-o}*buv2CHRK{H%~gTdjnkM5ZTvf#CCl2rc3@VV-v?ouQgGtTCAyK%rU}Ogkvpj! z@VDbR#*C%pf$eG3HS;CA;t6i%M(D)Ie7Ym6n+$Yip!(`IqG~uB9!}f?<7OYmTd}FQ z-H<8m_td5NDj2f^YVp z!wrf-IP}C5)E*q7D=aqQw7MZeRhpOyvQBI??<8rypGw$iqU?T)`P|CdXOPjEPaWlt zVB4okED@iGqQNJ~5juc_&4pmI?-2anCjpyJ&L(H7*WrDO1I+7lCaAy60QE0DgP8CP z^!MC^{YMf|e>Q{6bTRace*i_viy>zmKTq>-mFHqn*19+d=GWgT%Pks!CeV>Y+1zNXm3J#}K)2sE~Xk;}R@1}WRT&FUAw8+537vG7~Qc3g@i-lhA z`7pex8Ku8lLwijipPM~J_l?N_1(kNfC*9ET=|NoL{2$43Y~vZ9Rs4JYlKcr10?zv$ zZE{fJeCpm2wd72ww5!C$lQr-Z?@x|P@kF|E9Za1ym3Mi$qg7K9&j#cD>*rE2cDVxX zZ%;r`^CMWk%9Yg1{-FCLeq;FEBPd}k1W971Y=+x*T61F=w1?)vCCL;#Du1FhYTp-( zws=nL#}}7P5!!&!rdg!%oG|yQdJnB@olH}tbeS*m-Q?Vn7Q9%bfKMaD(4v{3bJs9Q zIO>Tvk`m~XIG)qGryTpE6*1aNL{P7tNm}`TUt*6hm_FPF0}rI&!vlNRYcvb4i&)^e z;B*vsjG}0ei`SbCK*#+k?wa0@-ukx~ZK1V-{wsQ5*(Je6%Df?44y3_sgLga&+6In} z+=h|I?#$1)kg|EE{BF3w4)c1WaNl%0H2iN5WZK%|rkh6GO}A0OOmj;N!HMkma6=6L z83@*nJ^a-C8^1K_LSlO`t@vC3(o>^JuV_C>zd9Grotub-KJ8eUV8AKz45B)V5W(Kj z3Ft$M@w1OOS9|<5o#NL?Z~VAVB{cY;*K9R>?;D8^s^?>tXA8(|tHT8XDeB0*q#@-4 zc*b-YTGe`E)$uA+{bvECb6n8foBs?6no+L8mHZmL2R*ygz&9rWUtiZE%%?xh%A{Hd z{@Nti7M{Vw5lYp%3Q-yZ}`jqMcas1}Qy&vIo{?jvXiggL| z@5ES==@*7g?#uZ+<|nkWZ6g)$EjYDlQSejIjIqv{#x`nxf(ga**b^5ju(9p2z@*>_ z`hHA8xkFwQw)$X*AMZpqDFU0QDa@}lVWv{-D-m9nh*~|`u=~hVhKTxD&OqsjeR;Wm;!rA8~f#g@Av~??S*x^UZ<8--r=jBM^y2$eF^;C zd>Ge#P~m>RJjZ9-L|GTbPQ-iT*->$O6Bv?Qy(aS`vz6QJR* zZFp8i8cV+SqqMM@jJpTdR_k+GuIg}TB!CPh{Uj0VrMO(1efV_!MQoNZ zA{P__ao@8lSYsWBl~Pa9fAum9Oy!8>GCA&W%>%k%42x2st5~z5TM#tPjlE|YfhIin z-y(GZ*YfrR8m*9kR`E5ovPhLv?0tlvR?Y+A-#NH>dOkW8o&(M8GwFz%JSV$_!nLJ} zY#{$O8Tv1R-PZMxIPw8sFA8Oo&-0v+Pbp}%#SxXdkt}Oa!Y)@SPF(B>iaj37?HHAZ z+pnb9iVg{`$7nI^PkaT-Y&pSW%OWhiokSY?gV07k3SRnL!;2#|aLKR%H|(fK+j?ov z?cN@o*k!}6m(9Zb$B)5re<()zTXU1ARzQQh7MEi5jwH0!;GAbAc<}3d_QB(Ba2h3%FXH9d@K*$l0&{>mtPqr&zlJZ{rh!l!Lcp}s_ z-)DL9C=!0&$j7a}%{h51bK-n?KF0iZ#M8wdn8>~&;+3mMAm@BN*1HIxdwV`suSuc>>cSXxLmNoqKAbac6KdQnMQ*_q zE?w3LJT_;;+=FslsZ1UDA*kN=1RwaEBR6ti;41kB z%;q~`#f}$HKUI!>a?t?)_|1Z`8oAW{@ocW}jTC#WPn}aZFqw6eNWzZ8rIgn#vANT4 z-~wk+SmHIFwHcYiE~u>`H6PPpi}XC!RcRG>bH5V1IC~=0uFfP&{1?)7OQwL@u}A1T z$(l6^vO)7cP44V%3pjshDwgNgV$0(iW_r^z*mLYZ*xum+PZvi((FH@u`TUpG-LYez z#82XkdaSA9Of7WeS$o%g7UF`K+pwg1Bf70vPfj7u_Ah4t^JzlsGtGE-FYlykTVobEbuKgoio*-`Hsvhhu;IgJtek5H z8{#GLzw5V2jP)9J(S&rE^mQz^R@xuqCLF~y97Op60i^pl3v7;!;m*H!Kv&Fm0ns}j z1$_m|RL#1U-&H=qfaxCeXnGqs`VZqG_wD#zO_behx{`E!5#@gQjAi$q0~FW#NNk;j zxx&dG$*M{TuJXAUB z2(LEZC9004g3Pt&aMr3fV3LpwyJM6|c9gPj9ae zqa=Ga&W@o*G2HRrt&@h$wI~<|oYI8BANs0IWs*yD1|4Ky!CeZwI0a!H! z6Q?tCAmr#7xb5+|^wHHZ5Oqu%idH-;OX2*{lw1Zs)p7XCRT^^MQ4qWNoGL&THR?T& z&?k#UI|xWDP{Y;Q*U|X1Eu=TZLCsq`TKvl#Qfw%3_$*6S6!?Sm+IYMuzL>d__?#N3 zhT`&rvM?*z0wyoX!;{A~A%k8<;q4BPnzIai+JsS|E{}RR1Y;l1bKJE)2*d6kqwgj< zf)(GL+_lmbV>X?ES6w&Z^tfs09OVQ0N8gZUL*6fE_lE|D?x0F}a|pNYD|r)fP>>@F za8X#6y;C0v4)-j9OMD9{&&59y%MS-8J%Q4+ zB{=TrT;>|O1Xe;ec6wwLLTrcp~_2VxnQ4rgXaf_XwQL?@l( zJ(Y!!JDP&ad-PDwYKWmX)j)T@45PX(6iyvB#3qYT`iH+goL^81+*Nh3mN<|5jyT*2u<|yeJ|Bl3-RDg(mCg2;r7MwD-;V+}7Fws1Y_^@+v;T0AdH`Vf0v1~L= zufs#~>tSqi3A1l!7Ce=-!iLPBf_xINxDzQkJ(8YRmVX~Sty*o zTS2@c_rlnRZ*e8_5Z*msBiJL!k}L-qFxGJ=La8C(+Zh8yO9dWk?uF>#rGows@@Va* zjV>jPT~LIsMWyhkJsrJE3t$rK0|xF>=rX0>5U5cFHZ6`+=rZ36oEreIgMJ7U zl;6@Rl^SesUlLvNHWNe5CV=IFPWpq9M5oPyg|W_A*j7*m{yX^&?e=y$Y@;RUI%bU0 zf7~%)(|ug?NCT%odyS7z?SN+0Fi`KMWVP&R?B6p56Fw98jn}c^kRpo4p2DZ)#g8%(G_N-#^N*P zE)jnv$@SEo!(`n*uq`zij6Mb8;+cU0!_eEre#>=`OqdKIl}%KAsy-O*tPy;!@&?Je z|7gpSN~--PnMswu3NGIFm=yOWGHk{>8^1o#ud-yVEK<4%-HZ7M08J(p;BS^;+sb@&ab2CefNom!9Liy zd@HE@oey0bcvqmREc8XDVP%ykF;$rem8Rj?G9jGri)+B4R6Cu-Kj*^Y#VA=d(t%L2dHc~6NAM@O^>85@to3k0BvqlAV4}*ZK_2B23`*bLJ99_7nn<*WA3R?bsblSBqbe_^u z;#Flv9`3gkWctYBzeRc|dCZmO)D=R<)`e)OUJhn5x-inMiPcw5(}1Teda5ST=))QC z?3x1aoGFI6r3I*yS}U-V@T2p*v_M)d0EdR23m!BVgKPSDte=q&e&-8Fg0v_+I^su) z9M{3MB3X9Lw{{#gzAH$X8o_7CL`lcv>F`HQiu=6O27kTIhs&WpG-{{|e+>VE)D^d= z@#Cv>`{E!-5$z;xq?@enUyL1^U*L%fe?vS{$0)~Gp}&tfHnty!)`rjY@H!tj;Jb%b zFBO47WuDRO>`OGY0;pC_9K0@8#W&N(pvB(^+~yicBxkdb-z~;{+#<{9kUB6QSdJYh zm*FG+p7*5Ikvh;Ej1;OTmU36IA}Eq%Z|Y#&`W6%8M+IoY?_DnV-@rcp2L3w6A0~y_ z!`1OaxM$=fX?~YN_ua3>i#oSa@2Ve-)Va<0-wQ=rU;$D~BfvJ`92U*;!^DseWIwQ2 zUeQD?9mg@m)d~;K7h;^3meXuSac*Pd3Aj329Y-1>$fg(tVmtLcIkY(t=qTUi>U>6? zjr~n)lG-r^qwrpLATHQ#hHs~5m3h|plHD(KK=Gk8z7LdTx36E09x2CZ|Mj0(arGb# zcO%$iUO^7<+%_e5FS9yp1K9B62~4vqp~}DfL9n%kclA|Z}LLe^b z0d7tmv@hTZsea*4oyNrDjyq#mZI9)|jL%Qos?DL_pJ>DRzaenU=M?57pU1XPD;)Qf z=lw_|qC(LH>i>I)-0B~uQ_I7d5rqiypy@iEb~h#Urb{rX;ww?=S%!r@rg(9i8Cmuy z6o!LO;7y-6`n%K_j{HzT?^S$1tt*Dc2PGryI7hAjt)ye!c<8UfVUmy&jg9|I;IF<4 zJHd26>>4kFb5;mQ@$@Vj#*W9d4iQqap&9Qo52-^D10y2 zKu7nLW2*53hTT`e_yz2ynp2~3%Jp`ldY}L%A1c6pk0LVN*@{z+C0N`oihKC1N`wApaZScKm4J&v5&PbeS0PW*GK;LD{L zyj3& zGr(&88k9{l!hpIbMDX|{PF<1?V(!-P089 z;{Bo^9G?xv?xwJ?I}%0%Jea}l@wC6w7aqxquu~6?MWI}!jjlCz( zP9zI9ZrDuZcip7S@EdUYo`zXEzXU&wTtIJr3%SJ4*k}2oK=3n5CRt_)*c*s*4P8&E zd66A-MlTWQI{4u9;3;6qltW@?8?xFRWxd)z@%C^OH66^QTfeV^-2O3apArlEd&Srd zE*rRc-(Hy&e&X-6x8x4rm6LJXy)`FBm_4(@Xff+0tOg6SIQ)Hw}GZ{J1Ty1CrfvoCOD z;%v;T5oZ?-T|s8U49Nf3jU_R0P%nK6_8h%{veTAAl;d6US-}r)JV`?TWu53Cvlo-A zr_n97@#y92h6?|TQ8A3?11Yq?-+FN`Fuay`%e{c8=1@5EM3nAJNnnlF(s zxc((McD0Z0xcUyt1ZSBK){V@p8#0(RYDB@)Xo8XCS3>2ISq4=K}OHnKKjBITf)gxTC)e-d7xkNNyO8 zX()2GYc%28*CK(nVg+mowuihEF=U=JL%GNN&1fRedK;ULzVGkA^^6ztxMz@nE+U~;7f^*CdO|0aoW)_IG0 z=J)_LvAsl3=#EnR<)^7`Z5i49%YaVGP-L&YG=`v=fv6$(i2CF{qNo1|(=A(yNlt0J zVArPxkR)Tct;0X?@7PDEJ>@fbyK@ECt|@?myL<$~luB3Tui zPG_sM!u!B#&{@FG0D_CSsx}o)wr>}_U;i5?YNWzYyd(b8KaW57&#~c+JSr5m2F16k zvlHj=j+vnQpb|S4Cm1_02|`ZrWA!PjaU>iQhC|u+g0Vh#d1Df{X z3_n4_oum)2E=(51<@3?~aVfq~h$HTM4PpMvIWX~06|5A^5^TJ|-w&rm5=pxvIAp4U zPj{Zjv_G=++}r@tck&?YUHKA{-h8I2@-2+;N_|d}=TEV+cZgEB3$EBI!fLG?PhQ_1 zA{x#r+@M7`Qz>y6-h>Nd`^^AS8Zw6trd-APzx{+2c7sgOpUfs(J??4WCg|ZAbEV(+ z)3PpxUKX0n%?c6)mglbPJ?Gtb3ybiLS&$%ie=j;IU*}oFBH$6INtSL67fg)_!b5Yy zA!kq@r=7mayw|jc^A`=FcQ}NvamSg-{1Q z2xt+*1H{klOH&=GwYmm3+eYc5swDb~pZ5;xUm`uC(PVkcZAMLQGw%8l0nJ58ywkf` zu(e$T#`T)8K~~XJoaZqvBV zCH~9t0SG&!-GNb9V#|#`e<_lZ*4^kt=DmZ;Wo_-o9 z0{ydx1=h?AZkfCU_owA9cFE?T`@j|0n(+*N>a>$i$!TcbFoVnZbDQQp6Qh9x0y3H~ ztF){}ine@E7jz2WK)c8!dOqzhxz6Wno?RoD@5{TqGegkg!ZjjPm=B|jENV>ti{We3 zVCT_DI(VU;=3MNcW-c2Au0@To^Wb?1tl9`Y%2$Y<;!DVRHiOGsGm+J~QH~{F#&8NR zj)TLdZW`Aq#HQYCCpJGOl6O15!uJMu!#D17>b|?Hd{=DBw`nIlx<6Dm~(w$RKRUr%9 z<)#Sai`Br>ei7%?`U%u*13|ng3kqgQvE_I2Xeg^pR>|gJ#K2^7r!@z~GH)@LB-F8R z_AAJKpNhTlJMhXw4G8`N+};acaQWi1Otp8k;QB3T7_F)h9B~N6h^u|1I?9jZ`djJo z8_U5fZ9R;RIe?4WvltzpZ?Jjm1c=bi$CkBH{F^obHy_I-YFfPC>rDWhcFAUveRQ#Q zZV&!+vchBeEAi^fx!ei$cB&`UgU|g$xbCbLIw-5j9-cCVz8VSP9iL~(Wp)<#!Ce;a zKX6CWUGoULL=GOl9YMGGkBEwrHY!>0oP&MtaEDKpdX!!#8h=i}t9|y^k#?SJA5Urg ziXG@QISDt7im|SFD%{uSFG1n&a&kOpESqf^hB4*3WXJAC{Pgw`$o{-W*A+g1=_0Ff z&fmqrJT)Z3I%i<+rx5h2pTn#zQ)6A_@?iZ1S=O-XD^=@#f_skWa(`QIL7U~qg)e?o zgX!}!%(BXcCaqH1FWg1iYmB)Uj`A>G?F*D>7{XL;2Rpu$Wv7&wV8+*eqT(IT`0aWj z*!?$HaB`)9SW5gPfx=7h!spr8_hdKfMrmN_{0?lh7UpKx27*;}F!Sc83G?}GCkZIl zLa9Y3xa_cDd}(V*hc{^Ai5-(Tp_xbFsz@iyNzA7S52lfw7rOyso`H;44U~AjgP;j| zF#9#{G5${uga1iz&7RRX_eeXWE=$9;x*@PuYz-&(bqfYnjiJxNZ$WEsYpJlN82fp@ zGP|gh_m=Xzaj~UJ+{Rbd7mYJQOwE|jl@@e9FS2Xnc zf~zKaL&|70y7Rx+54}pqjF;+|>N<<@n;rvOR?i?_BpD40MA$ibc?f8U%)f9YnOvlPa=I&z6KX#7wvvm016M&aY@?~YEnC!tE-O3wQC-M*w~c< zi+K;xtfihDsaK>q{>ogXQ4r`Wm%#cqBeKMF9tOC-gVZI)>>JkzTz}~`IX*WT7LjXZ zvSK$Owem5z83|)7-<1z|{RL&sbU4G6v$;FsU1YA_bz<@`j=Vdrh38B7KXXnp^XpbF zS>C+~ZthcOWuszY+PPfvJiwMDmLiTCUJ-=7(?NAy&xNkF=Zt^zjw`nbG`25&Zp?0AX*`G0pD`u9Va& z)BT{sc|;$<=%r)1kk~%*uH1pB@R@-8&3&}Yv>fMOOF)HF+PLA~4=8jSi=Xb-kV7sU zitxXuGNy{5O12_s#)ROm{l8&j@OrS>GK*YEn2c{H-k?F2x@^bI0=i0B6o_dl9{lhI zT6<~aIX>n}&4TecwQ-$2LyVsZEPFv2=z0Gew;TS*6{ArAcSvA5J@ zOff962&JQivE;tV7#J{WLc5Sw%zfoY?w*K(WncJA)r$LM=W2f5Qucr^YbIdScq^2V zDWRI~tsou#8QyP4@Vm%!tmnl+icB9788YDFNh*#+Q~}wjk97(?bj7k#TzxW2@MO;e zxJ^a4wd6W%+Ahl~yB5Iw?qt~c(z(CRa4ndY&R5^iQ`80A2z4+ z-OT0pn8k_0xa!(7Ixe?!q0-iD3~((GK#vXG)KQ8fhl#RhBGj$>G zHvc)Q{)nxYROm5fG1g+zA^a?NgDRP%LEpZ1z+o1{4yaJSdJ}g2nS5-stQW9}4%qgO zppsGqmfpC6Qyo2=0%4 zj_X{)$Pmwnys0rjqwm&~-M(5#q?6A>o%>em;%bU#Nyna=7Xw`f%}F)+EOfYmcVFEip5*(`)PKYnR_t+@Ts;kF6q3mzu%z zp%1@X)1j|CE|BKES8zDo0+kbZKRWLK9PhUc9p}6RyD@T@G=H(c?Y|Pi50eKltQ$l^ z+V?w9w-NKZ(zqV5m4O!F#s{=ypDncUqy32Kp+I z(WpE=8#Wf=Cv|~er6x{Ut^os!!y&703>sb-A^oFqnE&M|c8-5V>zlji7EcfUUe!xJ zF8N76%vOc}M2_R+L#eRACQNX{K1OgTLxTPKN*>Q{kHl1k&nR^`fH-)~f^RM6;4hfM zM4c@z zh@dH&BnrwPe&`+JA00YC3iZ-1r>+Y9$$DUC+Y` z=R2@cp}ovf#1zIa{g1kcO0d39t?*0Wa=dh4JbYZS5{B=D!pVb1tY~u*Y-=yYG^=W& zTb_<7=H^UE&stPlJQ=MnZX(hP{$PN7C5$NF0-cg`gsX@rlP^s~{e3RrP+W~F8Q0;A z0spqhl#y4%|KM!@Y;t46AHh$x-x%^{1GH?sDyV!snQTvs0-^F(v`3t0cOH$zGjpwp zM!*#qTo{A*v`cBX&upywqly2`8w0B>&(k!gxgf`m!a!WWzL9I)8!`8&NQpJ9WOREH2s=8jP=iew79}I3bn_v*)lZJA+v{ zb^^*BR>0;(fCr7QgZ(2hK#-o)vk1p5xJ7ivsIl9UTFBcA z8gv;m3ABEiU|V20^!|B5QngaZstspg&`^_iCnm$_;dcaIoMJk1UYD9|9m7ul{sc@z z5Ji`*g~{d_%(xE<+}*bkSmu(AhmW<;i$e2pU}z1w^o-q!+gvl|a``F>rL6!*@<6 zac-J&Wbr&D+~!ewP$Uf29O|J9d4AQq`PpXU`B`&rnhqQu`wWf6b6~+PK8tL&j0C8E zCoan`F*Akb$=NVla(Ac@D`s}__w7Ga!C?W7O&HIf=*hrIt8dYXd6lpw%Z1i#&xDTm zQ{l^(jS%?uCR)z4gz4v}3(j5D;)F4^w%yVol-~@kriEbS znuP(Laj5m(ga1ymo38hI2x{s)nCp-QOug*RRP9+0!~Ll=(Lfh1I)=eu*AM7bw#OU) zWU>53ZkH>Jb>nwTwiGr1>)fzaq8#98e6M1Pq! zfQOzqR!=;E*6+WfzPJom93W0FU2p~yeKYuRT$GG8=h+22{?X@hBH+8&3HxVu!s5Aw z)S=i4z4y&W7nvsN%lF%|^M$x=Q5^#R9fPGl){&^}84g`fyUOZHX7N6vXz)<520z8O zL^Eq5wLg~w8CkjTA&KJav0CuVrXKANoWb3-1dlkYa=n^U=**R(P!yuZjhGjL!>5lV z-~9$$w-y6I?sU#+ZaQo`y$nzJH$W}lt!lH61e0(pyya^Rilz?|sv9R6y3KgnVgTYIoV1IficTL3=?|KGdK*TRRrvD23i%;XdGeh*xrhfY4 z^=_!#>jvhT52>B<4;d-6K8FGjQ8V|AY*C4)Y@B+5;IL<4( z4#NLDsQs^%Fj1S||3**Zu%Mec_#q01w<{B!&Bx(2p?-+->ct(1G{kAXwWxx~Y& zxJoJ-2Im;yQ8t^lD{JH4iPO+_MF&c2C!+a!Ij-+N0nE%bh7+6al4}0v*|IYP7sre; zvvMzB%GkGPWITpz8p#m6Zm6a|3<|JhB!-S19?xwwU&%91qS0IAI<#Ls3=<}*Lvg7x zYiycLWOFrOl$}cz)ubW)x0;#3eqUH?9}Nc|#&8RT#X0wKo-1)VkIGNjMXIE;@!Iql zqW658**P6U6#J_M9h?R=A1J3efw@>YCkzcmv^e{VHP9)g$bL8Y413z{KuD=1`?*IF z?^SHa^bs}AYNjz>zTHiyzTl5(l8}6d{U%Yqrx8S>NACdXsLY`EKa-p}Z zsHhIlB(lDYC!SS-wt>(i6;{sVFKN&hV}-g9Cy!~s z@g)JM*X#`@D|5l}=ph{VS%kB9jo|YDaqh6&AilrufGXoH@X}auxOO}hZLNNT(HBkZ z8u6iBow|%+IG?MVaUBQG24Q>tc_vprg73(=&?QpUBx3Ip{zk=j)O|mIhAz*kN&gM? zV<&RfPx_#;FC1J9b0O+CriT%spg3);L&aX)Q)*Sdg0e*O&oUJM~vJQP-U&ZV13FX+^FDAQ~ag4^vn%pG)I76@E)jr zo6j8{TZ&qIFZuTF`*cC!SJE9ChtB&`@s05yj$f|FDMY-WXLn5IZe|*z?~Fu9ExK2B z_<$)}YBUogdY6N|Y%z$hUXCh5qqJRUDevL+fHn2&aU|H8I5p?<*~JWMG${%Pr>U?T z`FDQ&K5I5`a|d=F{>qHr)4|iRK&Cwr=5ALgp5@36+#$x`b4s1UZO?R$K20Qhy=sCNT-dL(rc4~17K7SYvf!$}J zyL1+I`ALvBi-j=2r2uzy+QGA4$~z+N;~7;zuj5mo)U%%$%+A1luFoORYceKfB<~@@${|*&y`w77d zmf&i?Ogu32G}unA!{u)Wz<2srNVxYCs;MoiK5w8Nmm}du^L{FhB5eER9duyt1zNn# z8MaInXAAUr&MtQvGK3Ym`|(4#JHj2NrFaQa7V4pnlN=okKY&I*tclpIrTDDvg5ZU< z8f#^rMJgs4!kK*+@pNqxW@#yL9@%d6z{}FIqqK{vTl&KuzCXS5B+s|r$TJ5+^r*yJ z-Z@nN5WUJ1sNb`*aKNn#*3`^F$GH+LcTIt-E4IMHEt+`r`b|*y=!L5G9QnxSEb{Wd z<671qeQnNTRj99-@m+ow_sR>#FEPcwZybvKd`=V2^SP)=cX4R7I?a6g9NX@dF%7)y zn?ytrv4@v&pMO0a*H(`9wlm;EOFI0BcV%+)TCoc?$;x_LcMXQFP_O?+mynz8g<38()xqSxFA(%u*bCEt5Vc>6NqYr{M1 zBL6T4GsICWVGca^JOwQ_DKK%>J)WO%hZyQzf^}1uphaK^B#lp*l~|CxnV-djQDW*Lym?$6>U|=x(Rdd+nOw$OX_J|@e+l5B zT|-{j`$2oadf0h36RyhbL8XD~^rCbUeIn!!FIH`V*`g+RH;(5bEFDjz6g5$J)f`;h zaULT2kJByh9$^QcO*=JW2^;=f1NZH(@UGp}Xej0bZA)Je$tOxUYh@JtCl>|ttyQr6 z)Oaj^E5I@H1L?ZYN{q^wI@Efz5&!lUlGyxRh_uUC=tc+p&h!9Gp z640=9#hFbTvD`P2OjhH+&vuXK}YBQY@oP?KK_?))BL>YlT3<7F{*TceHvg(~z~RUrL( zB#MlcNQTz8d7zsy5wpSqsBw-Nyoluu%# zLsZW~5IQ#?sOSy_@mZy2pVsQ3L2EL67}BFRr_I1<+p~D(vm7K`JdFCgXQ0}qULwR^ z#;7=b+A_yYaBhVcN$FSyWv3(26cx=>4@H%g4JE;?w^rmzTOrI+E@sLK1oTD9I~whK z9O!kvtAl*+swx0~lwSw`&v|J1YBEk9+KWc^9pprs4 zhziw4iAT3-f&~l5<}{l}_{$sky+zsbpiggM&NWY+A z!alewZw&R$^M{B3tl{beTdX@9NFu`nG33}5G<4j`=qbd(mK;S06gz_nVz#hB#gC-_ zXV27MoX&SWLuqfMEx!JK3_NXOz~-VGQ_-Rgj}0F(Z~ZeFwGjg*dRGrNJi1JNd0rMI z|4Knu$7xt>x|_dQT_Ac-`K;x?%TW6+j`XZKfm`>^!C8A-QE7G)ZDb3naVzgPFiOMf zU&rZ&01?<|)4EV*;5xi^cnJ$F$MKBya_kK+CG+XCviQEmIMec>@H^p!d;m$Es25%QGQ1tVh6L{y+Ik?HSm1o2I@-Op=&O@U_SP*#WTO8 zXjG{b9#l8O{%t9gK?KHs+Xa$C1*Gmx zH0<^ep(nJ{;PTxvDruWT-mm2ynaq8>)VmIMXhYlqXYx^ms3zXoO?;Ur38?1r5>zq-2`9#FcN(41}&1?h}}9( zcyz4~T{`9rK3>C+XLBSuiEPB+a4B|5!7~t7GGynQMp8@L2_U{zk7p-2^Y@0gSbkwV z#3Z-DzaP@D@aO{eL%=;)9oc|NI{)a;mG$s`6#^)SkUqQDa7;Z6RJWDmrpg68)5Q~t zU!A~Lw`^e4E|p3?;PdjIx1#QfJV<<*gX>yiVc_jyzKV(m)g{Mp>)jaasyYJ}%g1sXoVE+{ z@3mlDYZktn<_`@jPf?`l3fvnbi)FX$@WiE2`ZBd1=J{$vT=+CvT@i!&zQLsJWC!H@ zNx)&VVt}wAsPn%GWBQ`Y{2u&7B|pGJ4Pj`tNf=L6ih+6be8ywXFyTBV5ZysFm>F>n zqosck5#JE>POim=?pFji#y=!+HWnDiy&&e_`0mTjSZbGAP8yunfLhNTvhHRASn~dl zvtDB4*EcVE@xxnz=Y%@!75|GVyjS{S-8t%0XaI{=stfv7-iJksjWjPziDv%01Q+J$ zaM$|c;h=*gc&4&Au%L=a8vB5hbP3KHTnx2TKyQ__!w=ny_>&An*)@46>d2+8TP46I zK?-zEDsU5isNpARN!BdR8|M@_(2h=yzH^u4eqNWQ&t`AJbwTIRevJ%Vw0i;`)!9_t zBZ?6+nG0ISs>>oa%!6LRPLisw0YAU;yX=qFuvMjz=D*kvkfBU%j(?^*A}$K{$@gQA z_YAnQP>C*DU5dJV2Hn?K4i@+8kx4wG-ECY9U3x7M|Ni5%dsm84^vny`k;Hf1(p`C< z)FO~vVZ&^i_XqAym4PYZ5->^gCLKu#qW`??uympt9*b5Xo6TKtRq<4w>2v;niq8A5 z#{Z4urM*LoWR;3ak=c1)wDIU?I*f>)%U* z!IP&Sokf2Hey3ZXYvS;tZ@^t}A#C&#D0c0~gZ`@Jj*eky?kq+Oht;5J!BTGKX#mlm zZ@^cHtyu6Vi+p(S55K-Np{+i<%QM7wW1~(IT9(^@{ZC(Rhd6<~`^%Yl57uz*J`+&) zjX=dSA@HhRiOm2}_KEQ|e5ARM61RAIR(b**>Y5CWvyw=1Mrk`BfbTGcM1gtj6oq@;ukw=bJ%{PehBWzTDGB)DL3$ehpxKM} zRQWX^e{%$04mnVM>uUj>?O9D?{=S5YSULE*nfpFFj?(EbhcQ$A6~xxuAp_}8V3iAE zYD*OED{sOKehtjy@=3j;JkBS}?N8K|@wVMcOmxkmJp zmW&r0MX~$wE_zm=7DY9qFz?K5ESo+H%Maban!MdqLLw0y-z$SVkMqKxz6ger&xtc{ z1pl*<1D~xQ$(=%F-t9G3^vmzpL~wZ?kqjKgNxx3vt@kp^4?XCGtJ-tmzt9odasDdL z(C>H#SWHd7VfsuPTRW zF~#Kuw`AaIMjY%NamSrClOWJX`Q2s9p~qaDVNTzoY09IdsFuNB zJwf=Vb`CGaI1+r{`(j4@J&2tBmZ#fRfVwCE4=tX;{?!HKVW#G%kSOme@gnE1eJXBKyQR=@w29iklxN0dX58y*xdL;RC*Z@qA7Jm|T~uU) z1NP_$V)VVOB=_2V$XBj`FRA%-i=7XV`>zMyoRWZx4$p~2JlEl#{**q7`U&wTgOIIo zA~S9rf&TV>44yg&?51+^^;HN};?AwvA4K7&a0gyXC?%h)xf$usA`IHB4|-cp^E4v` znKfK*a&80Vyl}7a68tShjY_v1ScrlGamF*j_W=$aaKH%70{*f^A zfj5~GKMOM7Z-WWCKWO!labD%WeN>`Zi%$R1PR87~!zSfMl6b=t|JChDRZhF?LmA3v%xai5=Ln(`Rv8b_=dNDPj3Qzl2^jN`zFIu@u%?p zkr$m1@zQ8+mKd7I&BFbKb)Z$L536sbMm2^Sn_K6GS+QDY0uO?uOwTTwc*x7zNx`LBvP|ZrwKpl9n9e zm)nUk-dEjeWziMn4QfN&*ElR(Zv!715i`>|c_Z^<@iOxiCnt-sW(T-=ag!Gn?9AqW z6P-$P_ZQ>8x$m)Rts)GLHjvcSWA`Gb+mNRGcQ}+H1Fv!=(YmG_6PqdMmj*$9p$x`o za;_LBGu}iIL2`$$1hK7?$fgJ(_T#2Zc)rvC8ct^5`=deSvC`KtwfZ-#-RO(Q7nPuy z@CYiYJ>;zs;$vI$G;G^^gT(oBEXu={d}m7|mj9y~20I1Fv$X=y@nV>_Sa=kk#q8u) z8{EMopEQZs{eHUIJr(~x4nhAbm4vK$K^N-w!Fa1b+)|9h&5@Vjy~7z$zp)TSwdazX zK0D}AO=VPF?gY)Vy2#7hE8st6Z}KwTfvyPu48Q+PF2_3@r@X|L1cy#wbG+sNU3dmk zKLnFLqapApZ2{f27vQRR3|)Cjhgnto|IBhfwXliBHpL0#$lNlhFZPGpf@#ouBp5Gz zO(DI(X81t#ICUL1LyO|QWTNabvNviv!cPzUPJFQBa|P!YY33EFmS8Hc6F=-Y#<^1^ zVW)Hu@LSbLwB=*|-fzPgSegoHrzAo0UOH+C+#wnQ^JrFVJE-Q)!+?#7Y$2CTB#mWM zS6Gx@*k~6j5WSV|I=REZy>IGBji$#yf{DLyb zTCoT3IKLw!P4aYw#Z8njbV6F)Kw~1;;o8qt)Oo8Id*_@#ZPlHLagQI+(4F2eRYi|@ z2daRm$1}Xx(a$S*w-T>{Jj}h!xuNw#pjqu5$^Y<{C>=k7!J6G9!}kx@F|>lH^7qhP z8i<=7o`=S&aT3zVK)X&1hKJkW#E%o`YL^1|n^p{SHpbH0*!5WUB@BvcmFQ)5GZd|y ziB^__q&qHwDhpFCW3&~=b0T3p^8<*6)!@Qo%^+8*#ZDi0h8)8c_^L97?z_Ud+J>KE z_xd$x<=BYT;kO~;=WF=S=>VG96yS<83P$oXi-=CxXWF)F7u{YTi_4|P(SN%=PMXk0 zCU85)P=#)OllwXRS^gToJlM?&#((4~#T|#rW5T#ln`2wQ6JzIw$S`B2SMhSe1=QnT zhnpg%@Zw!H2uyS%E5nkhWAQHh`)M}QY@|YdD+{x_*89+WO+Il_c#UfLbFgtB8B4z8 z!z{;R#CK}|d@;F)#oi_$JNO#qoXtQY*$@5hccOGf4R@xC$H={pNaBtP?0nIITTciv zP0N0eoLw`)R4@*fEUhk&ytfB^|2s%db39;?(Vdv{QJ!QfPhk9P8i)q3h3{?`4hI=S zyuNG>=M7CnpTow;7LL-p8R6vdk_)8Q=PEfay@@E=Zh-yPvzgECDR?jaI~i;|#;l0q z=Ggoa_;lqz!liGp!ubMtRF}Z@eQ&YKuK;D57BbA%m1u3~1q~9)Y}bZ10NG#=Q|%(Y zm6A+?jWnzOZv$->E<~pjD=>6#FvwmlKq384bkx6bP+LB&XXC z@x^DnhfJGh_=1JhbW;krl@>zIoK0xrZ9)7`YoOY)XQZ$80d!rjqVm7ZG5a8&>t_ow zuBzeqzAptX9ad&5FF3%82Q2oikKy_EEI`*TY5vSf(IolzLJ*A;WBxAxj7IZw@aNhJ zU=}LT+-nPInfO(7j&VSvA}bj9c?t66-lC}U12lid2cJ!aR2}QUB2|;^Tj&cHvIi(n z)EHH5;zQ58hjfjBldVk$d!&>uI1Ps6^$0r=QOkWOp;0-5`t z!moj7aM`zy|L~AJcD3+95GGSAv3ouuY z7Lq5EWmr8kOIT8*2d&(BQb%_eUS(BLbNP1E^sffB?ds?!8%aBP`i$jMVGuc(25Zk{ zVZhw;Bu3~dwMrDG%Wtn{3#aN+DYXVtBL5Kr|Mk!+lV;M;*b3oy1eui2H_?F0x^1}< z3H?8>k~fndK|OCRz7Z%VHEO+d-p042(B2QfwKS6h$vg4n<=r^rg#QKAL0%o$@+7+0BJ&U0O zD$Q_5>@+4Na2$)8)9`L`H5}R-Pw%#Lkm#;Q9RD>OKIradd%q6iSb!XQ%RR=;%~RN_ zE`KPydmU^|6QF0=b@ck;1h<#J6c>lXO5G@Bg?69az9OfSp+xkW`OVhVmR>|lSMJ;_gxM^<_TSBl#K2_Dj-T@GIY;uB}XT;gMNfR>@Kf`@d@pi^?L}NZkN%& zf8wai*Av)vPzCRAujAcy(8VyJLbxI?!tlEu65AI%s29%xtHBI3G;YVDZE9Fh6b-=_ zgZWuHLaal*03+2Nf4lze?H!%oxI;8-!wK-R(#BPFx&@suRc zOjm|Q=ReR>-KS}Y@Oof-p7Lk5wNY73&Zn9f3B0eI3sY?}?7H)WueLmtKHHXnW+np6 zkhedM=D#O@XY^shz){+?;}QH7zen~hT!8N#b@;W1en5FP&&;|?K{4ti ze?+L5Dx}V0{TtPp^{rU@y9e++NSs4%k*9_4FWOoQ94>deaM5{!B; z!OBO8f_BS(_%nYmOcu`JH>O6zdi_4aT2BMUuoX9t3z3|Gd9d`L2HYkF;F+XDEgbSl zg}@8U+V~r@+e|PZdmB%{hmRSnUN0H)@+P;{8ex~)KPs;JmY9c#F?*v_nD&4gTAw6I zLtOj7%H#uhH1P4@kO;W{7=_VQm3*x*J7SCxQ{y2Ee=E0|-$#l~GAe8wOgS}CH#5V3Q7_HSN!_B5( zky}7^o@*jgf99d(Hbr)m1n1)F)CXVNFFaDXkEkqPOn#NjVwM_*;2V+0aNAHAlJxV* zG=mQK-Znt*zwW17*FFO`J?<`IoJQp4dtu?^3izFJ6yj~(5%V~CCZf5MZb=tq9Xro} z^`2Y&tJ|ya{|duIY(lFd*5@g^#KBUw{& z`PRwf(7V78y2~=D&yy~CEwzud{LaIYv1_~^iU=2;?dNCh>P4kBVI=<5Pn^Pax)){s zK@D~;lWZ4(+gdLZ(}eRhVd5y*j=qDore#=KE6=PbR!8Yl1)BZv9U)2YVcbLt7ijs> z6fSQSaXyN)9v9%dWUDY2QWHtt**q$icZ0T$h>^t1NhsRs_saVd zr|`_5KH74#kLY~upra*zn8&@x_ugF!US1Mrou0)y4RD!O)frUi zlq6enZmd>Y&k%ZgRt7 zKP$>^pxN?kQCRyd2&8K8dK*6Sg*Yzih3_wENx~cm57I%Nf;(CC;1_M*I-i!#u%%LC zO1xFta`?xu3~U2tz`wc+a49l?7Dp;FN=D|eI&&#JYR!c9$un@O!8MHGnV~YrrBpXd zLz!(av4!i1i|xsxrY7ESX-f%g4mgiD0)Nmg5i3xY+k3m1wSl@%3O^%xH7xh|M92F2 zVX1sGIo_c`+SAj?4u=O2tLg_*+xLL@JY}A=PZaKY9nCRq4e_(1I)C1UBrNvHgA*x>pg3VC zh;KWMHg?AB{eVnZv3xR{|3R7`J?xD(jp2BS%h{LfrO{KK(_nGhdrl=X0GN3Y6sy;R z-Yg&R-m1bF#JOVB#y~i7EWNyGT!@*oI1Ak5hM_Y*lZIv#;{DPzJh*Khlr_%6`GG&cbLV~XkNfY* z`CyL`;_D#!3INyuKN4vM;!iX@A-}Z?<+!@PxG2UX9 zk`qolZ%+=b4}yxt)#!VYf%z5s_}eWLb2o~!IX49{W6NdCyyHukN+t3_q)wp8p;Rhl z8N?(Hn<$=HJbL<4wf!e!oFjnP@*4(1xKV&DTPnBMqcJ${P&#N2410U zDHdq?I}Vjv-jlb7GqKzy3l{p^FMoCAA0dsWVVR9T@0hWx(a{fHh!Y}=Qanz8Q)o4? zRs!^!5!Zh?$9ZBkX0k#TPr)C*IIgp=#4J7EK#$wsqQkcVAVw($QZ?Ll^9-P!{Xj`T+YrR>7)R%w7h1VCszc0bv{Vv$_Y$xR2 zvBBZB2CPDY1@83}WoMju$>*z!Fmsse_)haF?6p1)rou^h@<=|}TUEe4>ywCJ&N;gG z+ymOAwHn7P(uh`XGJ4gi!5%M$oO)!z9+^53XxLh~_3|2?5t3zdZyJ*UvKc$=CE0oH zGGyxesq8Dcccffag!P|Gp={$E7^}&rLFQ7dZ(j_Fsl0->xMyy0`EgW`iGm5CTTxZ9 z0z^E0FkzM(4h%jZf$BQU#D|OF^XI2VDe|RYe#oCZzZ?f~5B9I}S4C78kX+}&}hAH~G2&9kbVorquD_O70q)%!^hk`_4c{4!w;WSi}8pI>6eJJ+F z51KdL#|2@cSm`dtJU%SOItB_bD^j_;?;xRfC7*+gQZT(W%O7|VtAV|93--B8hjk6l z_TZI=zZKbwfn4TAcr~80F2w{lOK>%e|9!~6!v0*d~tc9ObHBr!JC-CzB(sq+O;QdyF z6}z;TYL;(@olo_!=gW0+JbOB$wsD92X&WMqc-!>iJN+@UPKko$uRRozCnTBW1_{e z6?8fsh-{oNogU6{eg(es`3(%C#%rzi<#D9p?_+E45bZ{_*Ld?8(B@9|&UVO&zO zhuPCQ6EiNo#iQ(Rv=Wz~)x#~A8Zi;iR0?D5gaAlBoIp%`)Odnw<9PbCG$gNhh1)fn z;q3hoa>+dms#ffT>;)oZm!ROtf(e`~NV>^8hx_5p8l&QjT>o0fmrZN?s}kEVAO zv@mB~7IqYPad?d7&ZMd)gRXG_DcVNs-vBBx}U5*`zU4ntgkIEhbA| z!H}8tc<#Y2{^?%AxlLke9p{ofrmhSbYm}KOafNuzZXIqsu@VwomO|q6&A4_MA4#$& z&pMjp!B{Q^7wad~YYm@_=7i!Eqg9w}@Rc8=oJvQUUQyezmsDcKR(N({2QUY}@tx>ma4*RG;<23^I4p3+L6z>$pCTH{vib^ceQs)5n$e%lRFjrV-;wBDmkn6va1gf=DGP zSS+z0&*|8cNjnd~Y4==+D1Lx{Q`#^_rkKlct1-;FdsrBhiDi>6!Gz6uWcrl<;C4X` zUds9cs`tg|&K;rjYf~`1xgHH~Vl1#_=MkRMtqU-8uYy1D<~yCLx|?UJZVGD4#Ly~Z z5u;~Shu#+-lZ?+_;HJ4WEF8Dy_Olx~zxFr}p3BhGmoKUK>=OK~?t(8pWO1>(Hi|q5 zqFvrqobT8ha$A<;%>*tRbUK|Fj6J~Bca!K%ljpE>^%eLv6aw*{l5Ck#6nrSYh0~ie z@C(0}^eodP6*I%(&MXmje$7@OyQc7OT5KW0{f02bu!%16uA$#9^D$XY4h}X5Ky%p+ z*lt^jtJlVW{?t&k{oId(5Jk_=zeICl)Yz6438-IXhQZSg(xMPyJnJ(`3Kw0&d7)w8 zI5!3*2YL8zXHNOQBP($cm#KHu&x3idBk=p{gD9+gllNxzEnIRh5I*^P)0Ba!pz+!O zR@qzOW_LCEF3o`d;&Tn9&sUL4t_mCrr-Odv<-zp_$FNec4Fru9;lSf{&@Wbnj*f@m z){1iesooIsMl+EwCCxFYo@@u3sK-1V!w~q>vlYxbXCl@ege6ILvEoq@w?~abgBiX= zQ{I|HMrhHiU_-Xd;TX-umh^SZO^DY_0~6O4oM(9v@+x@bqdnLA5sjfLt#L5h!3Dar z%(#4YEC0kQUpmxh3DviBX~X#_Ob+=@(kw!#>yA49)%OC7-3BebQ!)b|k8I|fG=^gI z(Pu2TbcLH_7zWr)eRL`*mOgWI(XRQo?mH9wqA(cO-gYQkjNzyiE@WHo;L zn#VK0ngK8GYjNC%ROs({Pi!Yl$J3iQfBUIOa$0OV%+7hu|7;zK=hUXb$Qlo<8(oDt z-=k>MOBd*Whz1e%pz+Tb?FanICo$KcH8C8jI&wM3(nd77UxDK{gE*)8eR6O; z$4mUr{l0f^<; zMxRXOP<}-gs02EZ3vL3qFy$=DQw{9SvxTeKs!*8wg8aVoiDq+o%oFccLhhWq9N*fT z>j*|8pW92F3$~?8;-b;*?;%JW^P%#2i0>Bp^D;^ufkTolvt&c($&SO#5+<{wYkxA(MGDzw|L({QWk$w7(jDcwPmFIf8b^ z5!62_1*%t`f;t6l+)$W7zn0hIH8~~N$gd;WB8V)ltI^tFo$WUTOpdp-(y_O>H|U{`JqEsqJ@tYyQlVl;O9NR?bu zI3^0E|CvAGRn|^Jc%q0NiA!NuuP^iz+4DWp=RuvwD=3>i17bf~fz+Zk=v=|^FFF8L zPn>`T*UjNhY!k1&J%H+^avWgYp~uJMLjscq7>t<_$+a6<}N{m*ErL z1g3e@z&Y~`=lfg)vwf<_re;+vc(no?YR6$~%na172l6gF8#Euz$40RdYNS#O;mH%w zPE?&9?exVDrh?ej8;xt%E&~$`F&*?^ZhhrojWf*>(po=zJ|fUJIs2me*#3)?bcRg=kivEEWw%~|}5S3D>+ zJww_#zhIDQA-w%p0HX1((6~IR)_;e=@2|@xL|wp}3ZddyKa~tv2IqtS zVh(RUeoY^u$E=39`JF0GuJJ+ZyTkO;KTp8pcfiA21V)ceMA!Ui%z4v|zaCyiBi>qY z3n}D99LOhpL2Ldig#da#QxnWm!y(5zpB`o>fa`zfa5#1Y_zX&d$SHf?%kUdCzQ7p8 z@|F3bFUsliTh@H#zgPHg8m-`RVj=Asm1UnivqaCzN?2o{1FlZl^hod~Op(1#Lq4s< ziTv4gS4RbqgMma!MH7mcaELRqa>S_u=Y*2~d(7LB8+FM#47Hvlda*T~7;=Yj%-kGTvxAx`bNE2V#K6CG32gK)v`g z=#_kL?A_Q+_V*`X_x?oqQrV6EQ-2ZD<=wm|8P`!EAds4_ScQ=a+ThFmPnLYvgt+p@ z=yRv^UaLwuIBhuzzLBf=4H8WGix)dkev>?|OtK@}MxtTol$mI15JSCm1NjrnWYTp&yK6Ghxo79g1sLH)J&x`{!L&+9;aIIJtp!*)g{Smda6Eb` zdgJ((Wz?GMv>5ye#%bRb+4iQrv>Q|C$<^!GoyjLq>{cDxY^&p2%oJb}SBbJIGxb2( zNS7^Ew2x;ZGg zi?ROOU4Y%3O?;mA@OB9cVbZIc^u5OmG|KaX=g!u+n(^oRmV}|}kTN@U-dn2Ww~j2d zdPD`fHiO<#an7SHiV7EWi06Sye54{z7n}_sI}b{sX#OllWWED2Ign23&JWXD4)x&7 z`RJ}MJ5TL0rJUen?*NNL-F&s^*w-{v5HPDU8r)BBr_{D1YIRfx5|?mPi| z&VThW4bJ4A1O3&};1+3t=iI%i=}klS@IWC|O^zg=FJ#a&weN}b6Qql}m5AO_Tavxl z1v1%5kg?bl6|+0&D!F{35xoyJ4eN~#R2Rd>7H#rkP7Ow8zb1bqgGuj%X|O9U6rR|~=M4CBL4e(P+=I62*AP0zoL;Y= z4HLP})$m|@W*YcFJW+!K3ih@fH#T0 z}I|4Ltd`gejOe00k4fNZ9c( z&OIckP$F{ z56fRxg0{*Mj`P|Lk2>RF<$;}`UG|qBaxn^CJ>_;nMiF4Jw~;&ujf43bVc?x70EXLM zz$DFd8c`w+vs7=uQHhUa>a%~?5uFO28;_##x)s=~r_5Zg8o^Yl57_e080elN{P@!c z@~(}*bUAVKJQ0F-{q)#JVkbzD@4fQS6EPULPk{Nht%?`ipF;<3yg;a zzoRW7sC^!zR+r*XNb0}`}uUR`-p!*xh) z`HvLNn~W>kI5xqKlQ3v~5ImR8Ks8fMyj|2o#l5`o;U95!TTDDOaU7C6H;!VgnmiS- z<2di0l#1>iB=)6Rq*44nO`5%rbr+rue+0r|=yn85&-H^3qAqx*<`=Ax`bKsx-cP*h z$H?NIMqEF$8fI?{#xD+_*7W7H z9=_tyMbtV|6?~5AV*W!^3`AzG-y;TK`TWc) za*%#*2@aIq0}TsF#?03rf_>A`q$UTHzpjORjyF7y^OO`DexQ>3e(`y0KH;*Sw{R}~ z6VKAsnz%LJBgU`a^HMG(gM`>3&IdOi^O~z5UVR*xH&SHg0?x%MEygTux=)x}8Mvao z33t2KLU?NfHmci0^S2I?)Sv-u*9Ew1xg3q3Z3q8(MMPnW2mHuRgLJ1!41Woiu`K>b zOFfjBjAvJ=va=OldK(8Pvi*5Ew?ZM(BpRmo+M>In4nN81EqQ<9DaZw7LR-2ldt})& zSd}S<=g$2m>9f|u<1N?my+b7S`O4w8KE(LXoQq2$9=eK?aj``xIVRmlg7l48XSLt3 zad-wDVFtLFRVv5JpU+c0@dAc(l_7K(aP^EN__=KsE3&5x)fUE&p4aLFESh(T17^5>~n5}~f z*m%I3K41Ha-i)aLQfj^I(dk?a-Ag8#~9qxw#F zwz#&5P+N}mAU+q*XLv%~4IZY8X=3-!3>Z-~VOtU{?sG=Q)0&PXII&>5vNteV}1oJ8Tm7qjSrIm~@u|q%Xsg9G|Yng`q_euho-* zv-fb8qZMqK!+BDp$IvZS0I~EbIT5shWX3E;h1a^^tR0CtHAi`ZOcI`5ZVq!UbwTmU zNw`d7DO5@Ch0yET@L${r%#_K5qpBA0E2R;Bt``LPr8aPMlN_k+W#O*fa#+%H2Tq5F z!>#~FexLMg)-PfusAN3Bbx*kbLWK}!+$+PG-!yQ->qSh;s@F!V9&2G+JU~-y1x#66 z%~x7-1oZ>eaixtoTammNj%$d(^z+}zs8KjbS>(fW?p}5;iKYKmZH2Ld4R~;c5nj6S z5xR~9fYQ)2l$ZWMZKi&Opba~59-l|g?q9-s1ZQCN)c2UAT7Vx1Hxt3_0rXS22=7_e z8UA|ZcbNKe4_q1fMM~dYM$ho2%yu~?MpgHc(e1sfNyJ}{RXuwIsyYSOsp2y1YQ1k5 z{4|w!x+?;I^OrFBTJ|tueKA-H74y1d#p#X9j^%BW6&blB*U;WL4zfBWA=rN%C`K5w z{DFGTwK)?C)rU}aKes2l)q#hnzC~i#PCo3ufSI}B@ZiOHep=%b*gaGZmv5J$@n6bM z2>(qiLjvJK$s|^%yAqDBehM8`>2%-xP*|p^2TZ&HJofM=Q=Ppy_mL7Y4A7(vT-WXE zm7~;VjUCbHNg!shh*>;xhxbmz6RQ~&Tz1Ec{u^G!s@7Yf`@t4ko-fKcPXTz(EXEfW z0{A`K4$?im@VG-I)_?AY1BW>djK&}oQFo5=J8&BwPf6#yRlmWE&O+Rk+e)4Z%qB&~vzdJd4r9=cc35_y4pOIa z{$3^pFD7O}N*Dpdv)zQVRN>Em?$Dslas5{!d7H?&GAD1sri9+Ot^t*OlnV z48zyUSD>Qs0m^z0(7Gqpux#&S@Y|(El>A@OXMcC&cPR~YDF4B)yYmKpetm`YyDp=- zqYy5x^n(>KsqkS^F)yx_+h?wf1lN&R&{z75GoM9M_cN-@i0?Cwx8Q-s++O+KTR}K# ze~{QumSSR)Z^56Mv-H|tb#CvJ0>iJXV6MX*?2c5&s_Z=^V!aacHFzo+;n>j~OJ*@) zb>hq;-$Y2!Th0!Bd4Tm_d_eBJ1PE%0gJu{)yJVkAvvp*|16G9`*jSn7I`&8&sUj`7Y~bZacXJGaCG4c$LH?PimH6;m89%9H z6TK_`iQIe~2Xk9$u!rMlF?V#hyLKzdZ{u8G#qAhD>M*@Jnheg^1+H8cZ}SBLx36D= z8nGbGtBIl5P&OCurg?nKTG#^ntnGlrTfnj!X@b$9wIMQG2WaAE+2% z*7|wOOMf@+-Fy|RR72sl;}_ty)xz${C3tz`RrovM5fPTQLGQ_n(AbB|q;J|pw5$Ti z$IC0o*BN2h)sPBm>D=mfixQadQ%R_zFOm-*;imCBxL2J`wck9)mM53!esxVa+4l-= zoH&FPzs~{_;R_FBZa}`%BUE!4Lk*ofl;JXI-u@498_B1&%^%48?rltuMIYTCC&$PL zexZ9ObG+yNKOq074o#(+s7>$_qW;{UoIP93+te|}+rteJ=cVeynR=F9??>3y=j8AYny%)^B5VzjqY5H@~}!-(3GC|e}YKWtIVk1o0a zm&YtXvP~Wv>$M>GIg1@D`=GMU1d>iqfw@sRbZSilUKFw=Qk96XW*JWL+5uaq>!HEy zG7=(n3wquZCFzT#9odNvBD z&DCQRlH$?voF1t;nFA`vys3kZ3|nS6hyHlIh5Q>!M>azTx6G)amkNLKhC~ig$9D$o z$g(E9aX5`1^2G~o-f*YP!7^|pab$V?ZJN_lPSO`u(ERmz=-Pe>t}e_bqV4ZUP~Sz| zZu$kb1$-h3uXmFVb}WAtIS14v4!yXY!uF&f&|K}0liSO2gWh#|%8_y%QB`pLl7pMl z^FUzaEq3i*4cdE3aRSH2$xz7Te6yU#nBn{mtFlS>k4)O__m9*>BD5%ey>wc%IFFIxPdjieS$$LM2Q z$t=T4vhd+)7}v3p+qucY?Y=UU_cB^1zGNUtg8A_+9{MjdfWr4ih@beJ^Qi=* zL|wRGW>p?}-MR{Qel>xxV@pYU`2g?ik)1@_;W|%zdL`Hot$>)O7QDnih2pC?#(E`^ z6HSxJzJW>%)XNg|aT$!C16pv!&lI}4jj`!N8Tft;WoL~Yf*TbY7`iOM`b$6)eXkTi zFIa{O)@1zU+e;lID;m>?L8BS)U%#YRizf?`9*=`e$!u`^v<9u4qF}gC1V8_iLXB!q z6f3+6wg0KY1=GDWUi)A9S{RMGU zHfAZgQYylh-;5VblMTVQW@qtHRx1_UH=(!d4#SrZrf|{M3`2LwvES>8q2+@-Ynyii zdb2c9HOD}p5m*3g)#?CaPr-+u3wZ%P*>LP{H5?ZA$L)topmJF?>C1|yua>34v%VqT zE}vmKapXPDv-Kyn&&6WJ6iAyi%I$1yz^_#!<5y7k3qG`UI8 zcg3A*t~$f9Hk(P^U1z#^`fS+Yo(wm1fM!m=i0VrgL0bUl?F&|+Q7Z{)DKf`)r#;xO z#0S?G3NX6#If;BA0rD5;qT_pYa9lADKVQ6wA79szG5!pEJuw$%1nm^eIVnqKdc|OO zz7Cp;R&f5@P;}83CHW7|;E^pHLol=%6+Z{~!e|B*ZRjO!|9SlNMM+G`Y=;*XtqMI`Qh0k!Z~V_bao*jalN@yiA- zJ9;Do-*Z3v=YzHMBU1RbOp=-UjAMCyD}rd3dGOWvH8%UsS=qVEicGP9^^Jo!fzHq(BoY$sIA)sg_4bUvcd`8&71=l$HJ+% z#uri^QAj-vr}NC$J|OFj|AR65p4?0QMvsbw;|Q0L3g%w}i&?@b*)jz`KU0LJBeC?L z@iaDY`hWEDWKohOER1azhw0sYX8}N2Q1@gz)Ca3z!lehepf;U~X1<4gZ>Q4dFBrVA zJ{meU@ZjFe`8b#R%*Jz0yyDD$dcr{sIItk}dTwJ|t~aCib{%+(3s{wNw(!Yw7u*@x z1e#`-FfSzo4wj`uGnb3=`eFiW?n=;3O%e8si5M-H83Pf^Bxv`^5Il1(#-dFQtiq}k zxV~aO3o6RQ=Zq@k7ErRap+pckHWg3i|D!kA8LZ!+C@$4~L|z_hz~aX)FhA)u4OkY3 zclT}tx$z7h_mYBD((C}E-oA3pEd3!$5&pbQzdIafA17J>`{VtJ|>|5et`Bh>>(P_neZ+! z8RH5)u_bAPU<>!oe*P^6SH%6q?{AwqzqlNG?G@sG#s$Cc?!|*09(PFos-#~o7 z#9@JOHq88?&CZNIh0)vJQ{!J%cq$_Uvz&A2{l=$cs|D9feUkzHY6~#3=@3o&*9QYz z-s9NSRt$MDk%8Fpj8FL-tdf5~Tg;{I zR1pek94lVL&l>(tuwk3t)M5P3=e*&+ziF+qD$IAh2JM;N80B3F?dxw6*)yDHT1A&E zjI9xviH+jy1xZ$+yE?#)<3t_sZ{|ISO#_Xi^)yeQ%B-(O%RO4}V8DtGIq??hX|uCx6L@PD zD>4GF&PvMxhI7DDxu|;-#ALew8frB!LzHL~zljA~Z?1S_Zve@KO5FBG&wHB0N$`$Nw&GqDLFLAulEn6Go2myLr+~ zv+YZs4cmqfmO1haYno6kK!Kd~w#T-FTw1Y#p+1v1Ucc}tFY|{G%(}T5S8O;>_Xd>G zRQ+pY?V(b9T`b0a*>PQ1U3kOfJV7R1OPXZQuVuHZ{SzUXHOx1ZvrYy+D@UNG7 zBH#96ZyJ|9d{zqbY7w;7eG>*}O~J~w!Vs8~OEeX{(N~e{m1Lfw!QGK)JBeWjIpFZd zt%Wco_8!jKEam-KI-Z@frU)-HNX3ypHxTy^J;3UUv8J1#ts>b0b?`Ig6 zHx+f+lT>ExEz}fss0 zDQxa+=be;|fC~aK%=C4|zO*hbPuqwCA*Xm3_YMmDlS*jH`SmzvJ&t4JX~Ey3*U*@5 zfVFw&aN5W)d4B#b9lwWTvg_*bOw${As;Lj?9XmxjdM*x2|H`o1B1!lsql{NS`!%Kh z?qu|M3YWdrgD?4p&}SmTzt8g%RLv}gX@Vny7KJflvBH4%J}iY9Ds`A&5aO}Nu29fY2X!CkAZf_o> zw{|0Z{9cFO0v_SHHTmL?N*;8*?lt_BTS%O;qG%s48Df)01^Qmxvu5Q6?%eW-*=YwZmGRlW;La9>$VDMZU-mQK_ME?Y# zmdAKV5)T3Gr7O|YX*TRBRL8**VXPOG2D>X)(DvqCs<F)i@_l+9ou_$gZ|@im@KTz&BX#BB{Bhr*J=w= zrNU|Zhe>p$-+$2m#uaN!PE+@nme!`jSHZX|nMUl)#R!$v=sx0r^+P?RDx!8B+&IdaU}_TBLK+%>pU6HhgF zgh2NvRgUjjM0aOgr!_z3)5E4pWXXX7_+XobkH6&O5^i?YGRF-C_p|AF$;DV|yay*{ z>e9VyjNs$qtMFL5+8xejvMdx**E27K1^ zAq?N{DPm-ly+G>uU+Q2MLqu{W;hVXu;a~4IEMIY)A0sHi$YZu-uj4_ycxEs1pNN3y zvef|gWpHA(sDM0=CP|@rg7K#fVLiu3r?2E7TCR!S_;V1VEeDC{?JhF@oxu9V*NISa z@;EX7T#in=p3=Uo2iW%b1jb5^!#?%_#;iPmjZOK`J3k6+x2q88!bYC&XM$ zG$g71++K& z6`XN=h9ARn;dJFCTze`Xri{rjy~7LPWq7W@`RO{+9zh_eZzj=^co4Y*XX zjxq+9iP!3fF!#!3`opS|ObksRiDwMpX+SedX3DUicpgw-+KQ`|G{BH~H0jUE!m|Mp z^r}LK;8J2c?3^vnj+k-TyVy6p&BqeKT&w`})(+8urA46Sc0^!%n#%#Uy`W)VOQ_a$ zLlRI|KqA89vAOXd*`gK>`!hZ})k|`d%c?dp;TSA{_7RgI=0NwjG zq$S!0cAT9Lt3Upt`#wI$gtL2~`2*(-Db>U21^#gBog{mT%SqnSlE#Z4mgCWUF|g7Z zhs{xWf+pt(sD3n&dH1V_zdrR6$3D72*Z3vjLMp%!k9ayKa4YoLgrGsRIn-*O2l0E{ zIcED1xjM+l@$ok3$9c{or%YrYDG4+FdOB>TP&U@b!AQyiFd^95NZ?2R4&SnGURdc^lMnn;UtNcA^w71+I_2(&?P{wDx)+ zReBPO219l5Wc*z!`$G$sP31b$+BeDSDMpwOI|fysXHkM*u(oAsg87g#73 ziMtLLflF;0X?&3iKBJa&#{R3|J~@SR(Q(E!?d`5x52c*LL)prxVK@jKw|KN~Jh$up*@d$|e}F`ZiDb52I^5sp z3Z3hG(HS{^{oD?L_P$KKrZ=Beys1t*_3iQ3k^!*&GZ8i}%d=c)BEgtmv;=1lT_8Qn z;p4#w`f;}hB+P3e-+KKp+xZ^M5f^1A%5B3bcP>yn(QPpQhBXocG2*tO7gz8P^D6(b z@L*JtH9r;1d${c;uDW#?Z&Y=|lW+C#%0&%z;zxM5HYqSs_AjX_F{bzOMo?n72w%>g zk3G-L;b<1e%=j_`zC7E4A6}-y@?+y5;L=(6@Zb!sZta2(oqsTVdlPw5-~p~jWx&NI zTtHQKL3g|&J1%w~&bjRhdg(&!J@OJG;7(b4%1cpp*H>bAYHPk-``hR!v8tg>bunX~^$GKEgZz+OW_gCDy zZ!11za_9}84D3Dfw5AeR_BfVu1q`_J!GQCsU58cv@~D232VS?XkrmMo$la)k z?3Tf!0`1eT#OZ)OYjoNUE*r!V!!LhHrkD!H9C?V>4xYf2#5s_-a4K3#t%0TYPOy^} za(nI3O4{M)Zq;8V!^8y@Q_suxc;RIkj(UfZfFvhy75PQl#O~v2Z7*_&b2$V!9mX(z z3>lW)j<4^jz@l{;7-9E_?;G$;pzW4ROFH&&cVtIU{bGxYr}p4*w6`Fs(jJbAJjZ~6 zyYSyrj`Nf04KXK!VDmOJd^g_@o`($c_5=#3M$aXfv96SidprovR{~(N%^Pq$zZ?!- z*(%^REvWLh0~()g2H%5I1s&1c9bjK3JUe&|Zcea(K@$;hdK!fnYE;1e^-Ktw*NMWj zmcd|J1Ri>m2$dx^yn{Yv5SV`shq7yM{G=NYc%p&QjocsKKK7BEF z9`5?4fu&_?OvbxHx=Yp(`7KFE*I$8@wtP#WIkG?wpGM9 zD|T}}yCg;SzWGbuwW3#~AzPF^F>@oYV`v?!O>uxDcT|}18gT-Tv~3WtEe7vY71`>C z(|DUrPqMCDSKxf1fK+{?RvHo~p=Vz)t(X;s679Npt}q1BoD`UL4(MxQv;rR;6D6C+ zRYARUES__l#;93(V3CaxIm@;RSL442KCQFD&QxE+H~s}}xm19>8QOoO~CwS=Jo zQK&2~0h=>*Bx{x&Y7O3jo}aPMf%nMtBn230Z=~!qNgSzY1%sR1nVHLc`kN>6M<+Cq z(L0y0aa}4#N2Q|co*){3wHiVq6QRueJBF-g@qp!4xaHr8ytGWX=PZR{L6W$k(}yN? zC~;mU1#*=;tL?wu$qR?0BxpF54y0eeH!V8c*=ZE~UB1$V?aJ^xA`B{?#6#_jeJK5V zDwPP~gRg!k@8i!k+$>TZrd`{D`Dw?wyI>OBq%XMMsxSl}S7IEe?LeKSx^QSi7c9Fh z%Vsc#>8*Bc){?qli)pU4~<0_8hc^oeR?Ff}S2c7ygCR#Jc0ivDxs;r5d__X)qTS|0cVx zf5Pa#aNd>M{zS1q1_pvt1h3F{I>5GFe7fNxriXzBIHYgI|_|%E@vR_71_b@pa&lISy6HQOxlr<`?%+E`4oRconx1}92=9IzTh6U(RxE`e({9vI$4~qU7Ci?mLu&XE; z79QxJGf%t)czTK017YV99Jg=ot$>y>Dfxm z_=Kw@R5uN^XTIVg%z+Qus|2bYGQ>VBmw2BSrY!XMA zunRE9D~Wz;8zNu5uM^)QmM)nj0J%n6Y)fAb5C4U58PLac!Hjx*Cov1MniW``wI;AL zs)rN`ox`+(WH1x)1)sd_ps-{K+-tdE9+_f5Q~KATVe%bW^ZAS zLIa&RkL&1$d;(rtBHhF9$I=!H7)^Y^FS83JHfkZ{%eCe7(VQ`2t2K=MUA8Fu$N<&# zr0`@?JSv{PL^KN-GJDkrXn*1i%lX%c*2#W2<5U6DR9)G4n_fs!eu!HHeZ1+$Hq52~ zK6!p;H(fjU2F^6~;P>0j#MnWP{bE##6ZY3(_3o`?__j7vsU^yUbDYkh2lwHy#T1@s zm>1bvmI6$OEPF#uoHgt36C73D4Ch}?V|0zB*`IcgiI>$YoNFM#?lLN++6l2(Ui2QP zzIje6mxO|&wE;>lI!%r0imAL#EzMVU=CwO;d$Gt*q`q(uJdrp`l{L=5FULr@mL88G z4-dg9ri$-%Xbp(UG(nC6mzU=%4h?J!=p5yP*pXCTk!?K)SFeS!yXhFHEz556_$=6E zwGBV;yui#?3x|JCg*7kq1>dBeK%d_adginXH7aGnDDyTL`}p%r_(hyY{0iO=bKxCU zbD~RZ@*%mX7)P}$LC2^NCdK8U?1i@?Ml~;`&A2drOFp&?DyO0ix;ykoPG_H9lAvkJHgl}} zH;`1V!Uzq^aGb6`ME9H|6X{yQIWgs#;f`k9UpPnG#; z_X1C>`;4up($H?gDp1*Qibx3W#+J7ad9QPkx*2c7mgWFl#pU~2c1fm5H@8#$qsC0w<5-j@Hv}^4df{^0b7-v}rFvCWRO?YT z&hs_Im5IVk2VG2~PCH{`Q4Z(0zW_)4)Yv4mnII~(2R>C!#rNoq9;bAO-0Od^sIU}o zEhP?T{cYU@J7;*KTyzjL z>V5&|4rTN=>Ly_!ugJ0o$t1?-JKV2tL{BYAnAv;}Ug@ThmfBjpVJ(B-&+0M;8$J-# zZh2T~^&ZcCaK+VEBT?$kJ>0s^1~#=fl1kTcgb}?>T$p?G@YPas$z(n>6gd&!E`6$_ znFiPy1}A6M^4@e`2GMpUR9URZsC?ow?*~^C&;3EzHCD#&R}x}Is}y;~%CCUQQ9?7h z4`3D~2~(o|1+QBtv9EQxJVIL-9VrNewSH$Y@sJP`os&*Y+*aYL1BW2OSePl@sDt4l>H zFxs+8lL`3AWt%Lyd7bcTkle)iLsllD|LAxUd|RD!Zr|a2h2@|xI|U>3&ykEGD|qKR zoednxh9kl4AoHdSL^-+Bw10ghd|om4x#2ij8TZgriIZMh^}#%~P4Jq%j+5Se7PzbZ zhO8a;Nl#HWj7~g@2QEZ{sc{1t>sv>Q4I|*AOAd+pF9@zC{@_o&#Ld&oZ}1+eb1dEm zJ3+f53BDhk4HdnP@M5(xtgo00S0vBF>fZGvT*?^c3^$YOb8T^-e;KML$uRSF{eh3k zY4oj12&^5dBWSQ3&nBILfjQDFwpX`2IS*$%4EQG}=)jW~b|{wp zh_`OD7;D^1F+pKI_8yjGU6yWzR*}`%7;i=1F3X3-1zO0#k%I-C*r+Fm+6b=!%^Q7++$q>G4hk)SpG5^PyVALXHHYG(r3gyi-m~q39zfd z7?~wAXh=W`u6@%C9;uh1`>zduS=4n5FED@%J!vN9eJ*G?jKf|J9j1M*Ii2^zo5(&B zWd+mbvLfSLdBH9IaOF)qZY)`V@83P4-3lC6OgI>cTrf!Lb9*xO6xd9Ng}Uoa*u5f( zIEXs(kyPW(d^&wy8h!BW9|oO`g2is)jPdVYRQ-QO zi++b=vU{xtP8N{Xund~9fqTx0bVAHqD>h)k5OsN_6kwxq#50J_Rb0F!90f-0$ZoGmX5sAE_ z-Wn1&FBj;W6f&jB0djJ9FyP4L)s07~!hQ){RT04ZG4U$))-J%Pm`#kLPCPHcDVkS7 zA3)KRdUDNbJdGZnf&VO@!WYS#@MEzI+tf7z!@ly&wp*p-?fy6jn)8nr+!#;tj{GAo zM=hbQ^*GG!=;Qmi%W~aXNi@z9f#lvW?)&LY9&T=h7FRLQD+`66R?}dKg&C9Zw}ePC z4+N@b7UE8YS?r1n*FZ#TF)$V5*yzM|>Qi%pM$eaFvbo;e@lYQq+?0X;MZE;orB)EO zN`TuGw&NhjQaXNAn3>w|PmR~wE-2&}+xZQ=k18b?_qrC>$`p|0V8iIn zmt!y8&A>AT)$r|6zO|!R86LWqLVTX~(~xhOkmTyhZeCH1kGNb`ccKys{cEFhF1taB z|1r4udJ@buxP(zx{m|r(5G?hc2^mfg(d*SsXsgN;IQtbs_K+-g^-D22(Xhlr7rSh4sA=+w`Jtz#U+{pb{^@Xy2iTla}y=nyZd=o|{(FQ?O& zXTg6*Vz5*DEkrabv2Khqd-l*7h>VGZhL6E$W1)%hCssD)#x zzXWBci{QPmn&5Mt0wzDsgNjX|g3D3j^wdi|+B0?%zQ+b&t9U;yHrPeBcqOB)^$lpL zD57ia`yss41HL;Zz`vaw3+b~xJf9+uE86X8R2L5x+S|b%wwPXMki=hpsYKH29Bi;o zC8I`PRgHe+Jp4z(?b1 zd$_xBDlD{2fV1{af*ba0Fq-3H`G_=9L&<7%wo<~`VsW_tn*o$`T|@PAl~&VBrjXOW zXM)$gSG;PKWctzd40&>49qLO@L-*QKoGV+LjQ5-mzN;V8&ld*io>D7nwRsTk&3X(6 zxvXG#ca^o$mnMk6p@1_b)}m241CiE2Tt6|JNA@Y>@FG#B{9poB?`LRtdJ&GoU%Pm- zzI>z*{t$cSr^80>F5x?0l+CEmAdz}}UgeWuf${wc3{8(F$GBbW#kc1nx^)Oomn;JN zFjHXb)Ud6#3S=dg(8+6x;hoG4GSk*hu>My#7>pR9XKptREA8a=%96O4NkX?<9|?Uu z3qGr^#2&-77`3^Y{Qc|#Hcm=T67boLkn*G+cR^ zva)!pxR-R9c*9QHWAwzjOXU9a8*p`38|dtxh=Z=N^yybKxM8xF`djC~K!SkiG;-No z_XbdY5d;r3zCz5X1D*)D0KMW}^wn)GR!L$BC{|sdl3wYUy8j?(MT^4v@$11pDG7~D zx`^h$ZG1Kq1**l{aF05`J0CI9=cY`3i))DKwRmXI9i`nIsQ%21G}NCgAji4fFgeC2 zN(X1K5v?D{;{R6hl*Z&C#v_xugm&O%6_ywMT9H{O(M?5WPi1nsF7LDZikPz31#9yY z$bst?yc3n?xX{&^ju?uf=OYV_YjF>^iJDXWLlZ&L^#Hw?E5?tzxEn0V4I){!9$oG3 z@NSpg0 zYoAcsJ0ENuxU<_FJM<45u|CbKhQ|J4EO?wpn$zaul3AB|Y47gvx6>#*7!`+KSDWJ2 z4I21NG8UbRpMlY9NqE3@1(e6V;%BeBXRYNoOfKIvW=}nO3%h5WhHat-U{ffHN|p|! zz*G)q8s>wZ$~wr8;pSh$;*4X$PBK+fg)Z<}OHCw8XsvV$>Gd@s2Hbg5E@3aa^nAt} zlEyg0LloPaFB7#Nmh3MVe|-B&2ut>xpn>sJ6mTpKsZL?Io9qL#HjSeb-~+kz@CF>< z`d#~8W$`k-ZxdCM89++x2|I9!o=kRwHoFTXK5RERzV|Wx5}bvtO)@z7(`Bx=6^7Qy zDLCxb%9}H_1I_Qu1hcIP#H2ljDn4I?L+9G?nVK>*n}=bb9A_HV8_(Eat z1XiM&b0%pe!m*J}cr2qDJhgNgfoBdKzE%!8BoN{*bVBU5Lc9|j3T#3NFQ-ZYC!Q$* zC;PL|8ZQjvR_24lE*?z1JB6Lv_ZOUI?!e{T+$J!M>*#LHrQwSw;?Lj_dQik3O~RL< zQAIOumpcI2KU!##v>Y?*Lo^;R62U!d)X{76Oz@o4#8>Tkj6Zg8d=)Qw<|jJQcXv`? zmXvA3|YnN#Ko$~bjHvr{;`a46n`7SscVU7??b_MP={@M z6hf!gX`#@XYML~*leUGdM#HfMDD80!Cupi;=&j&}I`I*U*3JG9Wv<&q9it&U^ zJ09F(gYAA=Q1?rhePb{lG7I+L@JStbf53q4*d;Dl9b3Y6wxsY)TRiqAY=+ZQwYWWX zB}nQWCoxm($TNBe<%a)(da5$n8k2`pR}GPDspq7n+!KY?Ya)AjDr!IGK9{>H=rEU0 zYpSlLQY%xbnc;c}6ftKvx~hPVy%bq~BA%W+orn_*LtxHq9i}_hj&polLvGb5f8X-) z_}DUnZjXCGRCC^fyXOI-Cs>cOr<=1%diy}H{U83W%K@hT1FSey%KKpNh;n6Sto)x* zJkfU*HKinJ<}q~;d&q!H+yv%NHXnE2eoXgWE5Kb1?qD>(p4ZfS6PIhrF&V|`P~N|t z95S!q7Y-eOHFB1CW9d=qonDNptxn|Wj8mXoHJ*g;+Y4#0gwdtX2W{rRC6n4M;kwmh zi1=5CRl8j%!!awx=L_+kE?Nu^>T+@YDCcTc*nkJJ9^-kf&)lB*4e<|i$6$+KG&>Bi zb884%i1zbhop0il84i#jUrS^ns;Nj{G<_71Oh$vApnW#ijgCHq+LPU^&L1A+?dtIt z9Q5(WD|fcz_Gu!_iXs&}{zQZ1U2=qQuBY=h){2z8Xo9>(j_?0_06)(+p`L3?@v+@! zxD}``INYqrbc=q(lSeEmY}o?ShecV5k9(luks&Za4@pE*Cf0wAh8w0^q5O+4Rm4)V zaMNXYy($y^n_dF5*As7>w}Pz8BPi?&BrVR?Y?)6Z?=CY7@29^&9(ShdzbK3kZcJkN z0iEdJdsko^%I%4XJ-y6z@pj7C5f3F@W{ty6jN0h}o{n2lN%|#W7FgmMXHA@HSqDD( zH=%oMAM9)6TspQ1u(eTx0V-e6n`10J}Y*&Oke05m&HNpaShbd^Sykq_3@H=hh&~!Vl-!q2Z8{KHau7}pfap~4`Yxj{1Y6#*1nk4PFC%#+I zK@=yL)6EtuFlxzCtm}C~YHs!l?wfkhZ|Ns7%c7Z%Exe4?q{Vt)awZyyCF8U?-SmZV zh2T%ncC1+x279+T;|(q^9mtny(IE1Ee<)QzuJ-pa^ z3@n6fVY6u>l@2HZ<2m=qwW*=VD6hc%!42fWIt^y7cnq|@Edf7^d|Gqj4R69yJvv&t zm9CWH;a(>p+FT^UdsiI-cP{P#?KuOOxV3`h4`zcq*OgtkK^UFG&%nyc7VdYB2@WZa z&_@oOcW_6cRcL-Nd1d#8N+h)4*3eFpy!H$cG}J()h!N`7zQ>yq)iCYoIBc};!tdX? z?@`xO`tYh6Tz7Ql4dj^61-6SJCG|2D4M~P~A11)Ib6o!LfFirr&l7GpK0)6;U);{L z(ML;^AhC83q&7^PaCG-iOZM3c-H~EeB=f6n{b@vdtx~5gAEPgdIm9L3nBiO9{bUBEBe|O zVO;D5x-)t@o#8KQ`K(2RJ!0_hK@TJ~a#vz41EmiWxX!G@P`w^NsS%5@M*wCb=i1HhDY z(rKTjvsXDMdPV3=bQV>o^9FAM)2P81x1J_nO3Lu!EbgqAE=4b=X|i9qduC^2HaeDe z!rIQW{IUuv zm;9g;9<*WroTu&Izmaolr5OD$9Db{HlCid{G_fa!>|ZH^ZyuXrT~`EjP3HV^l__{h zM+TJQmC3qav#FxNHgt^4qH8AnAlnzr0Q&{yxRNr+GDq=SY_+|`Nz6Kcf|4Bq666x%`Dfm+8 zI+cvy2G8sM;mZ>zLGgeXTNC=2^j@9;I=9}?jP=??(dr(Z^&*7mes+SRddfI|W;e!& z4^jJqP*Qm{4)-|c;7%nqSh>vG9C_=LxA&dd-iy zmQ2*%33!n!i*ZkT4RHxsgfi>68B6^Qa%%B8B4h1GB2V{{87Ixqd2QtJ-k50^}Q6CH~m0y!7BrwmFx-w){-r=F{EPQpO&KT}CQMqZ?Wb#Ay>VHRp$2m^lF0+@8y zMNsNB{b>yU5RaaB&+QYere z^_UyDlG`orqw(CGNvsdKyMrUO*YBr_Me-o@<`ZdN5R4CJJmJk$Scp9_({a~e5Q%d; z4oZSxbYE+P`KQ7-MsNv{Nm;clA@-d@p#Xcbz17xWR?-YvA?pDyZ$BkMmyHqWG&s%+QwN zoWiDDW}yYT(>&mPZxrfp_eR579Vp+HPZ;Mp(0VZoEC=<$-LwHrEluEbH|O-;H4lO> z+kj5kJe2B|BR{IAL4xZ^RGLGmo|_a52mK-WM!6Vf_L#o2RDygBAy7EOF(HntQN_yh z^#0CA)|Mg?L^H<@(|4O)$LesktCQM>B=dq6Hj~&Vs@(pn9Zyd1 z1?vMPz&@CPt75+sxs^>kzMDPHdMu69Y9CfV*+V9^MAC2bHlvMqHn}5ff{Rb5l8O2v z%tR|qXc7D*DjF%&s?HRs2N07`z@VR!a7;}J<{2czIz>_V zmD)tH;1t%+l|qI6k@!H@g2V}4!X1B0)GB-|$fNyGSy)3;wuF%_O3n3F!m^+_MS=T2 z%dlr9OTfzF4&{rN;p$0?Kqjj}a4IGNzNvA2kAHWFvgciFtqrBoJ0=M>bMLBe8>GQp zumOY5C!@fR^JbSlr{xn;u-kbS+HZYFR ze)BliZa0pL^TVi-f3WtjpD@GwdA7vFU(M@r{z=wBDr^0a-T1JKX(-P`_BuO^Ww4CJPuU-+i+D=Fs-ol z!st(Sc-+{Mr!XOdSpK?(q8{sc=gPZjKt?!BY-;5BzW8h{{$!X&9+!k^Q;z}gZ&4)$ zE5i4D1mSh_!7)$0Wn;%iK|peba!Q<39?HN`J$ zZj<4arvxz%Z(?R#HXNV$8XJ5yVD5in%rTh`8tWMWzc^OQ6?`vX&I&=$Wf2g0ltdi@ z6QF5Z2vPlX1g^Vtvsk&ggg?gdm%^vPh2{geRxc9!&F+C=-xA)UL$laF<437!UNG-g zj5p3O)ko?38zHbDP4L_>7A{7(V@K2^e$la1FihLg@P0=c?SGkq6WiP2+A;z8J!1~` zTnz$iTT3+4?Wd!;(_r_#srXXO3eHc@r@HHesOgd_+Oy;YIdOO|NS^$RrMu4)O(h-n z!a`y80UvOliV^N`;Iaa1JW+jtEf{>9OSJbt#M8C-Y0Kww z0{gMDS%ubgN0N){1vqcL2cA;7fG_@gP82hjqEk>Sd49?O>P-%ShWk%Cc&L$g%Oeb{ z!jj>d)h&8;V=ZmU>mqAcyrP~p&2&}XJDxyS8m7l)VyX6K>+WnJM!YisP3`63Sc)~% zGVT-@H`}7~<5y_1v=#deD9EnxgDncT@NTjhBYV_}>b}1VMoDdS=KL_;jHDrs@p>L! zTvC7|P9fOBaL%4TWrC#9CQ^8|3eO(@3h2!3p3i?H+g1KY(RujQ_`Y$RhK8iHw~$dv zLe#mhV>GBpif@!cR!NbJD72@vqot)G4cc|?>qwMDT2d6!P?1CtGJenR59sykd38F^ zeP7q-^L~5KORt3M%!9D?EAYmXG5XT)9%wh8A>~>- zKs8B_`#NbOrzc01d}zIcH@jJMZ{|~|`OkqdzJ!qV`PM}A@jI4@q%pL-|BOaT3z#1L zBUs57h7&IQppbTjdADq21#LfwYKInLjzc94{VvG;Qd5rV_d{@rz#&v(&h3krkK&oa z1L$us18#0-oErDDK-5Bw`=aD2aZJ->JKCgCzB~_@VKf(F76@V1!_CmM!x!@%1-N6( zJ}LfpDZ2k+SP)NYX|?EfJbotxqnIwPL-IVf%fNeL5PpVZx!s2tMV6DW5)ZHt(}1{( z9prj%3GO!1g>CQSQH$AQzEd>kn8tCa?z?cP)My90(ic@@@=3HXVhiI*QYTI2t3cwb z9&Eqij6-e7#4hI?-B8T}`ztC<)n)VyJOnlD#4E zCAlYb00ab=Ls5-BCJCQI8MPZ2SeFVW+x_uKT{8K3^;NpgoBKgmGd2c?hfLzr$N6W^w-wJ;dF!weU=)7W^sA6(53 z!r2SFux_v)H5Hzr=KB`%oR@J2JQ$%ugg{$sD6yNK6c%MkJbFbpn!aLY zXYH`Y(U1O;-9%5WS72>1%O^#`q21DDAfHB_@eT6I^I;m?*S8By@;*{*U7N%Qp+v$xU5UvSp z{-l6$fj8Y6eh`|XX0oN3Ov6>#;I0!AST-QdEo`WwAKOPs zOr;G{E53;@GDC^{au@K=EkKK(nV2VM&duNZ8AcB{vZ~O zYGTM-IUTGKtN@XINv_7<#oS|@7sTv9A3eAECPogmfZRRC9rm~h3O*b|cP%O6qEd=~ z+ymj7Qy_%;ao~({7S6vsn;S7pm-FCHJFa%$gu;hAScS#UI11JQq$O@9`(lkb8aqBE z=l1GDeRD8peE29ti3gM0`af7}G+uL(Pp<=o`b-d&jwb8)QmVz9%UIqC*Fa-^7!3Cd z;oCRy#G%IpEFH_h((*1kJCBhD``^?$R|{8#KBXf)k?2?BN9MdtBQl?#lBt--kg`qh#|DX)cHj4lf@$xCA?eqqkzL>q{ni0CBOtE|fXR)pN=9BI}3%LUsfUN3Pa_MXz zhpgivBXgQDQt1JUKR}HA^PwxMK2Rp<9Z_(lHGlODf;Cwk zpndTITngJrEbNLPFD?!5y3Au2`v`OIjNif!FR#$qC!~;@)Q`0~f8f**47w3Cz~kxcLA^1HHhsw1ai@-xcPf8WY~&=;P`crd9@llx19#_q~j2q zFhHWj69EF-aQQV2Eau^(&t4i)o@@HhF?|9ba}tT)EP3dcE2QCtleFc*d1znB2V*75 zu)as0yQT37OFBdww3&C%zZ0R5q?!ghDH=UL(Klr7pXv8bMT>J$dFxZtAy^jR0( zmf^{$T1uToL8)&9f9}+S9lA5vXYQNhgU)G)k*fhA-P;@=8JPB3pR++L)>R{y;fk16D$$dlx%icY@CH>b_bDIv4553Ny5A z=(oSJ@cUgjW{w79@UI*a+`1L|R0Z+jX)X5NQ|D!_dYQ+son_tVjsXh@@)6l41=i63zY>!K`s@X?4nI^!N4CrQ^5WkJjQ@U90xPsd-7DqFds$tn#eLU#n3P!CJ^zK_R zI&s7hRJTTP4%ioh(X&9PANq%*+xfWJ-_)TteVjITuY;__FyIOwA$Q&TIDPXRuyCd# zE34xl8DCid_lsVGXp}7)N_B$vR0XT($PF|!sR!07LN_F*k~CfqYGKcCK68Y)Q&LHg z){uiGLM3?f-7|ROT}8}JNO7xP%Cr3=@?dF$8CD*&g}k;s;9J>8f2FHIu^B%GO}2uu zRxtHGmW+#%QX!#&fN+;s`NpI)%?9*WiOqCUC4I z6viI~vrewdpuP9&z{|u7%ITLQ84}NKU1Y3tV;FeqSDptKVQhbB11c_bLWG*N1b0FELk|gKum_aJqL4 zM%(Y{?~h-Bx#vR29P7-<*#?zdqHPtVJu9G{V#dw~}4}4n+z^T29vQmuj z&aQFXujO4mz$=G$XZ?lH(`EQ8PLi#;DWA%n_<=uH9R{8x9z2rwnzT>3U`nhRP04(U z)|!6s)Alx|Gn}h!#^Ly*ubpK%{{X9+%#{GzA z>Xq4VWwYqWr%CeFP>joj1!-2DAY>>+aG2`~IGGM{T3-uruRWh2YTSjmt|gYmO=O&4D?`))TwRVDw+Q6vJ~B z*g;oEF|p7L)rA???{xzZ*F8ZBPpw4NXQFJ&q!(y2>`Tp**pSLufD>m%Ve6dbxUS>| z(Y}<)%o6j+(V|zxDwtvaz3#@s#z{DDA&>d9=F#`jOm{o=BYF8nmTeO!0e$S7I8~p4 z%GXLT@i8BJtw17d;^X7eSU@4h{bYQs5VLCvV5wU+9RJ40bcgT5_n|DX*?R-#3PtM2 zT7SSZr}fwiO%kBkEdi|j8CTwfFRZ@2m|fU;xH`4Doqi}PN6n5%;&~88TspI4MzX>VGCJKiuz`tsOQ*F%4p8DI%Ij)t5V|$r-^@U=( z|N9AieVNHb`gdYLGUF{gpvB~?*MM+qEMEK?1p!<+w$AH1TBICD3%TB?u>Uek^)JI0 zb7bS~iArkZ;RnGzX|&he83k4fqosxh?2Zpco4Si=@2`VPDjs9SF$=i(`VsDPsYTw4 zbBVOWTz2S&5=`&^%Gp>k6GlfYVDt2K5IxH!AujUl){AEF#UL0{Vz#5=V&KbKQcwTS-~Lf_6_z6 z2y(>b&EWOf>v&>P9Nunx1GCJ&^f$1k;v9ko1@O2X2WgY1GOO1%Qek3mz zR8hlQ{OnJzLHJtmE{vbmgWsQ3xGqP(;qCeIFtp$ROnw?7h9-~nHXJl$%NO_)Z`Ntd z(H`g29UX<4@m|e%LUfAyV94Md>sp!{ zch3b_}(H%CWcq(`dj5(s%v|t^RKg`C%lFWXyejY@d z@5Nxj5m27WhwHupSih^2RIS+!SAHe3Hcm7VM}8O7-{nESKWE(Vs!v(DD^H`)5W@i` z!tf%D50c!pAZ5K4d;P~kCU?<-H=E|NKYuC0BP}IxrgjMJAHTrM2YVn@(H?uhoTb^B z-gwuh7FLEuLby2x$FjuODp&b&#}?-Ex!^QxYkH67gMaZC8GWL~k>d?9;LO;e{hJB~3o@Oy#io5BGoK5qr(MNMO%*@dsckg&n zjp1b2_D_lXJ@F5HtYwM;E8pnr?>8f%FZH1rC2_}Z0gO?-gdR-a@UV?0a_)E1(!XD^ zyw3|JgF=yWvJ~>gFVOIP2Z8rE^Uf8>;*7YggIHq^lD{jI$quG*3WSeCt8p}y-1ZNi zBnoj`R(fKJc00~+iNZzxbtH7{bIwtvY;a|=Es>pNxC>^pqxMU%eWMEKzPvE9H?0o# z>bAqK;(T0@JV=_lOF-+`7UD0I57GmIOb=ZebcG*NSKXaZw=oO*qLMIc>2`W7b}h)a zZ-bBaS0L$SBMLHo)Eb$KR8Z;?2!J!%&>8H3WdZQaRg_%Tj=*7o8@PCxKMY-3j$*%a z$ePbltgi6}5EILUea8FA&gKBn;`}47M5_7^vrb&OIuf5I`cTGn3k^|v?3<>+tm_~A z;cIX-h?H3qKC^zjaw-C@GtX#Y>5A8MzQSbg2LN>e?#VhISi0mt?D8JSHriqjE@CH2n8%5SFxn$M(bQ3~oZJ~$RP8CLNAK+$px08} zsnxG^nEF*kK|LO8UbxZ>c!i7VrpbRPH;BNwSa8e!L&N9%hpQfhfwkracDCYEuwK83 z+%+*{?=pJ`;-O1Pz{+BXc#wdbn|F`}C!TP2Xtxs9Vi}lpKZ2*$6|;uz{t>As3pi5e zBOtV+jb`jsfHfbP-pF_x`S8>SXKppY?S9!TkIFFID3=PFp-1S>f|;!5njf^F<_2|D z_r>dPH{tQT@~R{Aq8Sb&!}GNgWs6A063eYtcyzlN#5ikHt-G=C!a0?|r8soE90Lz{ zQ{m4hYq)vp7iZPt3!ol&hNiae0`t5INbB<_DQYQH>iHHBc)FV1FgqFt?)|26UmxI% z?_p4TE)0L1Jb<51M1X#lNL9wz6c!{i{`25|(73Gu6Q>>leWwXp@(dR_l|viqqRF!A zTlk{U5WhThg3=HCXj|b8thD!J-`#F1v)PSEoQr`=OJa~3i7;#E6x8YtqL%s&!1jF! zD@2{ii^j9GV$&Vyxpa}VE+3?R28yV-m*E&Yg@D^fcQhFb0f(j}sEkWxJx*zc6W!0? zBCJC%Xd?S&N5Pjk4y(f|n^STklHB&n!I|A6bjCLoiY~LDT)T~&8Uhu$Ty1{nKTO`oC&V;qI`x2fV`Jx~Hpo%& zJl&4b%c_!xJ4`)rd&AkDqoL9p`BPL(IGm<4QtVeZB9r5DQZ1UbH01Vn0c1BVS zGk;`u(mLI^+Wst@9{d81YxAg^YZmK}z!b6eTEv}-um`!@fHPXNz=f2;wKeK!_`QIo z*LfWZ5-+kYj0BKSO?NzbBNDc2Kg3E)VTk3Q4NWup$?qa}`uUInDK&URZ@C<#-CoD( zo(+ze^lm$jPRwRMH{S;ROD^IgpIz|by?=Gr)%VpxF~WGbD-*BgkI^@w8&I{w6PG{R z3uW6Bq3L)DO}5`fwIvVIebNumeL9F*kK5zX7h#aX_%_ zH^Os>%1$Y^nA09kLj4)4f5izzmUvSG@r#%=cMLW*ghP?z4(v~lV69s65=S~0!}`Na zCe~i5GB7g?g1*&b_46jU#}?$qx|rii4@a24(+rwe$=|qz7zOKZPCJDH*t=$X&avoYT2%Ir41#gI|1J&~1v+AmYw&x)5B^)p`{?Oj($l zpA5e*#L>ardEE8uHEGk&3K)#K2Dcx_R!cs93rgqQID^a{etK6G?5|~fgKJ00R!qmp zOSvTZnH|s(aa=x4Sr0Wlh}l*duK#1^+*q)f`=P>)<TuF94be)!ljmOof)LeC}Ny@|P zOjf+o;Uc>3k%!RdCz#!3D=~kU3h#=Cn7wQx8hX4Z{5JBS;(*xxxrO#Ga-xqtM(DQ5 zTcG!=1$CHtiJxFPeK0EkcDx)Ve!UCOffIojz8vG6Jd+2OXHKFi#|s~HRYBDTesmFk zhUEo))WT{3{tY+?vqvm&>!R&U!Qm}O{1n4Q-(pmKVrLUMQgsAOk5)r6_ZrTKEA-Og~Qav>h;6k@;VTF(8T_ynw^f3wy! zUek&|NXi_b{{f>DUxbs^3{aSy}_xbevMTun=^< zE(7Okg0PJt0B#Li3IfiL@g}_t!GS5%YNG+Bz%j6W){HBbZbkpsy0|WN6)f{y#ATZt zMS%cCNE~>AqS+5g#fT_MsjUOfoXza~g6rs>`4J~qIfBh^8Sag0b>g9a63#qJr^h!e z#ZBxRtg##d`|29ey`YB@8)Y~ubs1e|U&EV6eX;xOC{7J*AjgN7&A4c}zV)*oSF8MyTjLAi&A9%G zlR`zV$5bxUV~7J;#_jj?bPf4)`VqrCK$ z9TJZ?%iV{-==BxoRjk1+EovC&)kLp#41>rr0Zhv80>Mk65K=0HSG6{?haaeM%OC9_ zRbNZt+?Wz5#+E}{=^N7D^by{)Y{JuGrZDj$73Mptp{*1; z#p!e7YPqbC&Mw+~D1`CZR)UI06BW?#N4-z`sHVbR{B|P_i@(WWx_un^Q@@*-?w$gU zLKar|$e`Tue0*|(mz<3JkLK3Q;0j4^K$kPE;CZ-<%-#1E*GBTP6MxUs^Z5G2u*9s%S_wQog8&&w(UIf3^*Kn3yGvLlR-AffR_QA8yKFsek zhZS(pfv$Be!&fo&aIVrFJT@biM@w^FDjxyGMklO0nF^JY320_8gq9wnWTVGA^p*LK z>Ya$jqb8{=^Sf0jzjP)B{M4jg-`t?%ffDZ$na7YVQZ--s_~@3>Lg~aXxxmv%*--_%wjiQUqyAolTlRi6VW$G zC;F-vU?PeIuk5~{vZgLKVqYw|m0JSg@Dk7We8QcMSMa0x85mP|0O?J<=p?#@%3L8R zpw&ua{zl`x;a9}x@-O7u#r%&AU!$8#B1y;1U=W_MkUetcH+``_nK*A*1FITO`kpQT{#b$~7rtEG3WRiT@^8$%XM!M$a{xPNOVe0H46-766a zyR@4b|4a>abIgIOeDUO_!6!1-;>he6`Pk#%-{1k?4iI@BN@lZXa`g&rm zYM!;&4PhWZF3w$*kV%|EX0RPKba zX}*9+3j&#u<6)Mlr~ouM*`Jv4TBE;@1ThzZ{nTK#7Rcl$^=PF&5#V^vac;kgrT?5n~n z2aM2XqzB!DMA(PedYoq252&*)9mC2uF)pJubgS}BaC~%_v#W9o9uHr{arn!)IAuz3 z+gDR?oOIV)AT7< zpHvBQWsS)*TnU}3D_Kr{iNq+GnOCd{g19CYCOxl4MYp+ZzZFH8Q(*)LWJgJ2e>ZDF z?gZpJtC8TM9#~vdMt64v;^9!0ieF<-NFwT#v zdF1Qn4|vV-FfjdvnNy)qFr4!jJb)qSK6Ka%dTdw9EE z7{bph;1fFsGV9DfJj9;>H^&)Az`GDM$#_XCK1!m{t_XDddW%R+WfJ4>&uBOQ5*$`h zCx-8$s;68+(Q@1Xj@V0cLMj=SqvoCg;a zlOIE6bYtjdPTMMmUvu4yv#R9)QN8((Br9!YdS91dg|QH1_Bt@U9#L}M?*<(A*#Z91 zdF0i>7vw|RYZ~$}AKwgHk%WsipyhZK9+)XtM+M#D%-HCn|LN5x%v!V;xhgzVM#TqQ zU1y>C;t=wMVF&MFn*%lLCoBA2u|m!sgMY4rJ2pJzXa9b3@pdN-Rj#6|$!_qNUQ8}a zgu}<5S0K983mZZm$RGZtD4<}2H?3Dd$->3t+0kb5>{dA|`C25Y)yl>N#sV ze0xU&Hk>~~Ud^wtO@FAK1mWR%qcanKua-r|LAr+1n0sSC1 zcrCvQPrvy_qQhE=>jfe3-_c5I9;WK2mh0kDD=DZ`62V4=9FX(7i3^N!VEONZERQei z@Kr9WPh)|^6U)w_kG_TgZ}Fqfa} zby2Z{&rtkTDe+O%rTYovK71KNZQBN@;u${>PZk8PdcY?i_OK2Zck3BkCR9`L6{m0U zLpWd@PTfs`zEbvvDG$bJH>8dAlUaCJFOKw>NYIw$)%cU?8!h>Mhj@mqMxlS-NJokt zb;;7A_52Aqc&QVX=|x47!@o-U5+VyP`U}- zC%Xyn{90PSSOAYJIl=*>XGFg2V|9nyb(C_Sk49d#kaZZb^Zt7DcK2eQF^1uEb;2P< zZPsY5D;#sKq@EH4B#LK&yW2f-;A0+Zwa8k+3QR($rX!?Q=Njx@`;&G#++$cgm7Mh= z7f9aQAB5JPgW$yFaKo^Ub3HHcLVcOw})Apbpq=C6p7&JT%P!Wfo%+y&u7i5S=_ zL4u@{VP26q8j&V^`FRDZmsY^uAm;uNNkQ$|b2`n>a7--cqkM`CxCV!lfzlCjU}zjv z%8WpJVm93!_n3++^}xxo0)29B4^6aSS|>$57!}BA=zSpf%MFieY^9;? zIoQ9Y5pCqU!KD=GTg$}|`SS+oGAS3W@*vQTGDn@5Ph_Ra5^DLyo@Vt{;0~U-@bcw3 zj^HgnlA_-ThO1iW65$(I!aK@xwNXc{U4gLUi~=Ojh=aKbR`~XA6Lp$pfnI4Epnmrw z9Lk@~8Co+<6}N`sWXo*${dh4NU&(`LpANJ4dB5B9;diYu%h&;aAvFy4S(Mk5D zd(JH*Ew%+@t00%$=dOX|l>~dQ%c1&nUGUd7guNC@dbX#N0b@PB69UT>(pIFtXlmWY~qHr35SA7Ltu7q9=ALH_(u!r|} zMBQZ|*DMy9G&TgB=Apepi^*Dyf~9v_DX*P5G@ZDJYUggF_k~xO7&uP7-ue)`s0wU* z%z`hymhdU9gd=-joX95U!kIL4@>r(;A9=lF9s9`i(LjpLk$nzU{i*P8l>^K?T1Onh*Ea$6`(H z32>lhpmIDIBD{l`zGOK{MTMhYKR@F!@y8Hz7xE>ZgL@(U3m)* zZ>l<1jYad&Yfja|iq55dKbLsgT)`&; zKA2VIjjNIZ;jY6?Z0l5oxUzKcq{%d%n}d4$__-Q!oA9uXKj+|)!*H&RpS`Ov61Yap z9n#le*G2doLQor$GS<`JS-td#0*OP<4>U-!Vr;+N$=k4U8(Ryf(W8?cKO)T&4 zr$whH^n0quU|tgsF>0G)S$gtuw@1o>?!6&6P?w2s`hSt?es9=S`v4@iZGecIt8v6+ zEs8wu z@L?^=D@5S@b1BqQW-*Jg`(o-B9{lFpf^%&Ah{IuN_B^F%Jga{h2JUaclX5ALo*;~-G9gxEx>YvBs{UDkghh4#xEg^Tg(z*VpSmKM)-hUa|Uw{eob`}J+V!CH(r$* z=D4_evo0@s$=Wy)4c!{!ME=(}nZ55G#yEJ9KdfmymV>5= zMqt|*lWos-z`okMWD}E{cwo^$!Wa6WrPMRd<^?C=hn5CzHhD_P?!A~HDhX|Ct5_R^ z^KrL*EgkFrfxns)V4b=I_wMyzYH&Uk8o!*zb?1JN=ux}CVZY32MxEfsQjVFpcZ%mEj;Cc9ANT=#a#TfE)^CZ9L2VvbM)aeQ4HK< z2L+GUusymgFxDmwL?iZbZr_VUZJXs7e;c?OofDwyc>&D+$x?TwLeW$3+(x zf$58VX!Jf9yN+seSA2HI!PZQ$K6MFRBtOIK5K*FfR2k>i4zkX>rsJNIpGf@vdeZjZ z81$y+z^&`4Wa>{VwlwBp<^Dw|^X~zQG`M5VTNY%`C?=~y%?{N*YU?@WNj&-P-Up$F9v9VD(+rd-j{ zZ`k^_2~zqF()+&6u$7gAJ5*+1X!bo$qcuo5~I(Svu-E71FUG%(|o8h3fpDq5Id2^MmUk6NXMTr57l&NLxDQucriW{q(E5iDYJ1KBow4&Im$wgE z@^&)!FkQ?mxr2RAl38wPqcnR^h(=W~99_u+@HM^@q&*V#4KmHKvHXZg>j;lLH3e!-++bOCS{QaU1&2a=9T z+@G-`TyJ4%{JlB|FFH%GJ(pj=eawtm-h2^O%4L#GOZTDlWEZ%8m4XAl&*6IJZtj{T z5wKC?HmtDiCC@%MQ~Two$z!J9d9t0C9d__D4(GMg;9CUX+GbdBFBxa-$p*vJ-C)mj z`KK@Dk*%4$pnV~Mwcy7waCcn+{|p&tlv*iAB!uB2=85gcpHyR zeCk2=#CSU|WJdXk}jyBc1pI1Bi)@m2>=CXH1;(is z4=In9l1E*>=r(&R-fCp}T=K`s;%nNtXWmQF(shJXNlP+!m#wItWuFNJOs8wr%28pwvcA^BT{|2CRco#n9na%dH9wIdK1^JKL09GkYkC_HWY$4FOTE^?rt5E%ORm9zDkxK_UD)Ap~EOWH3ab7lwr& zVB2&YZEI`6J=cxUujCfoXq6*vddKmX&JPG+n^lW>x6x&O#ps_`PxAi;K;eoVIQ+Vn zd~X}2W5kYiSN0$5m#cub+jTI;dlJrSy{4%u_u$2Ky9L|a%+fLko z{>QxB?YAFuUgcSULg6ReZLkbvpBd0WzRQFsUls08i-1VQ5iGx!00T#kP?`30*f3xP z%JCB9sIw|(`yp#kD+`8X!%8@{xE`hi5t?iM(S?>n;CjQGF26T`!=IKQD$YZ#mkXfl ziv!777LEr_&0&9U+QT_Bmxs15TLuLgy{xMaEo9TrIJkLo8jj`X(}7EdkLw=wbbxSj=0=J@+CIqaI0Mu+)g&&O^T$hN&j-ZI5TNB0RwQ z-U^?z#E`G566~`|A#%qB)Cg6=>E))xagG(^N>O4hzB>vYiIE(gei5h~&7g*(GQcl0 zg0ksK?0rwtXjfDO%t@%hvw=BaR(gk{DjtY4dai(FM;#SYup={d*RsEQeCBw5enSn@ z&%xN6Npi||J7&j^Lvl+mT!^`YqK4JPEY2IA=L}H$`SR?>50{9#bSn0@7Ele`NM>x# zhd(V(Am_C{z8H`rTfc6kp$s?jquV2B_>l(ce(sQYtOTz(F*|u%UUpaGCNA#}A*$Ft z!Q^&lgJg>W@C6p)yG&COBjf^GwY{mIzyYc*zY(kETBF@b3otg3A%B@n)k9^*J$F(A zFFn=;*1ut9C+ULY&#h7QehK47O@}VEe`NYBg{uHnp6Ip^>n%br3>EU)`69nGKXOsoHE5sPeT&m@Qov& zsDd(e9OnD(66cKB4IEDB1&76k+~=xB>|OUIaEa4jTqf{lb2Z&^TG1R1F<=!7%HDJlI&@0>8#o(ik*|ENgY5 z@3NA$D~W8}9EA`2FAZ`v48Ty-CNX1Suf zZaSt)ydfch8$jyIK5SUJ7DEM|lBSiaAZ8i>KMt#cwvh!+GyDHcD8`yCE;vDJXqIR? zi2q~Gd7+_XS+E6tcu)hbrsje~>LJ`9I0Z9obMc4ZFxp@CLti(2a((nK%pMnk;GjU1 z+0BNU^Bg9J?uveku0qDXEBJFJcTr${-r%Kb7Lx$3zcGJoF%%OyJbbvC>io3D&ARL8{P_R#NDxXj zAF@u#Yrw9j+7NbpA^elg!GAuUcyUmYz1xoIQ+}Hww}0HEChOx-U0DZ{@8?raSqAaj zL&@14*I=~10KRV5<{nK}#J9R)Y|YEk)RHnAN(Nt-nVE7r#nmT^)(WB%vDe z@Y^=w+7+hbA`wEDymP8v%?ceN`=YX`J?`gLw5754l*tq3n6d zBvpDneo%V?X6au@F!K)Jk?{fFp#r=RaSId@7;bBxDnx7*f$H;IoG7v7MD7(}FZ|n1 zOr2`6a=i^X*5e0tds1PRZ$88IIEyO`ov5=pvxf+HNgQ7v(67CAk4!$|a=@2DQaC^R zsn{p-$H^SC`VN6e$~-7h&gJ~4`W%<%`qSCgzku`3n6+e}62GZFr%QHr!|2+t)bG|9 z5uKX=TRuEuGBm3(e19@r5f|p#ee=gxpG4UM+;8O3vUVajD-iIbB75oV5{;!O@)p_h%w!_P7S>KaPis-!h=)s0--cTuP_ak3$j-CN3-8fFq}?Cjy?R`DtZ>*TO{Epn^=%!K9}qK zwsGxOI6(8pTi~I%8V^5W?j#?-k;~5-@t8vokuYC?HWMmz|DZV51k6PJQeLiV3O{GP zY!dCV+(Rz@XOB@aA#irEgUO6|L)O@J+`{z#8b;@Uy8aCIo5X5dd|@HWh)*v&hJp{nbMm!N zW=HIN3bQ8D(fg4i!_wD4QKMuSzu^e^*Smp!joP4Z6# zjF{zMRqR2`<>^3y;%X2!UI?vWYPh-N1sS@$6pB`P(iu)?;bE&ET;;7r==%ftWvPASuGb-sZ-|Q+*O3Iw*$8VItgD@rhJ7iHH4J&jXj- z7JzHZ5rR9Cab@9Re5@1&>OMSZdn1krx6Gh^wfZ1Zolm|k_rf#Q6_64&4Hvks`ad7# z(R1S}RH`E$+Y`!(=WrxsTWzLtV>59_?+x(eJ_P>n!Q|Zhdl=9zr9rl$rmD#phKN`#2D8zAADf)$@5hlVM|FMdNBWI zCtxo|*b<0P^&_PgzW7{I7TY$8k$&$UByI8}k8p_dJyrJjfFB$C3ifYIaAIB&iMSuexmv2szE<%S z*VcB^Q@?&u;UBkX=ba*!#yZY#& z=QLdNA7|&BCt!3c8%u_pNeOdK=H4ElAAdAan~UAJ?Z2-7F?1e|RJLIlw|7QlRwQIp zs3gvFpJ>o98Y*RkG)S9D8ulI~Bde56Rw|tH+%H8TDkBXmEz;7S>igb5f%Bf{Jnwy9 z*YAfvdjFEs$)_zI&N1aAROF&w(H9v1u0jp9FJpxJ2lDp84tQA3&L$i4;a}+q65U|{ zi`%7OLYJX8=M>ZD0VZH?u@=?t?dH3fTSKymIg0H+4dp2pAgo-BnU%g1#5`2#2@XLe zy=hF<-wKEpmV>JcCgU}wc2pmBhnMCX=v1A2a>4yP#_U(e`s?ZZ+|7I#6}d!m^0Igr z@>byGRz4|hO(i;>3vusuQ?RSep#NDEqul6pGG(6yJPa5C;U&@h$VX9V(2)Q~b5!Vo z%%c!BgTc}f3fj}d;l^5Z=A_qUI?lVbI#epB>_GWVwE@xdMZ8qRNU!L(;?Fm6=ZsNztgT(Fb13J~s z928tGupWhF98O0GHhhW0q0I`AIXwe*4M1!m1SdO}V%oR|zhado3>(COhUz8Q_gEee_nkyqry}qqyKkt|3c^EN0VoTc!xbB! zi!&@DAd&YIb0MGXXLCh6UnL;(HXDM??D(6%Er;gjbb5R5C+c%}8kub?jgv*zS}r-A ziih@3!u>4w$${$vbEA#KY9LM!oMMB4BvlVl)X&C+yOsEsFMhzY#v`0>>WWana&YH23!}+dIXvNC10TzUVIap2>^la? zmQUdj=$?SPbxJssn{DaIU&>%Kd6W~&cASDkEJ54qFBSU9tdE z`|%5@AL}K8>(4{(#!!5rV@u7;oFR8W4QI_xh3h#(pnm!R{k6V}3Kel!?tCbi{Bz;= z>&lVq^Q9Qm$EV@8fEr!BMGMroXM$7BRdkw&qw24+&~x7>j?tS1aLqTB^`fZJR0}H- zwC^U^96pD}H`hVsvO-cd7|inj>rfx%aiMJt2x~}lL!MOP?xI(zsWSXi-+ ztyy^x(|ncAYMF|RWIX8*iB}v=@rP^_=l71EoF%_&sbJ~{ zPFLXyG_1&^Qnntjd}M&-kB6ZS%SBC({YR1nPw;g~(gBN&ln9815iLfadKJ|}$u%on650i6iXPw>hp z*jcAQ0zJ}x6Xv<SX@ayrkl8m0RY*DjauGMa`;kNKIvPkgy7K z^Apha$4y*l(1I$0vq^7L0k6uuYBk+~Pgo~N_VJqxJ$6o^b;4H3+YfXSzSllaIGjLEytdMqf}I_U`K#`BBh8_Vc_ zwoMbb#nVB*kIVR7Rfe9{Ubs7rgSUr-@P#yk%S#v{zb1p_zUJYxn=^<-1_HmN4#sod? z+`;Ufr669$@VE9Z0^w=CI3Rd}-7Cs6vj^32-s4AL#+gf2y2kU9UWsu1%}db4_5z0O z%mQ`QVpKAJ0Mz|GDVSOVS&gSr%={+VY*&n@C(lJwWM`81D#_H&Y4o7qEjs_sA;K?h z=6q|I3U?MNV5naoNeMoN4vId^`W3AEsBs?b9F0bIIrdDv%QAk?I}r6c4rCk<=C){6 z(zucTNa^Se;E4!g&4w%Z{*e-sWpZhjOwNTu@I z*!&KgFTb`5mD$eE{TwaaoT|mC`C5g3m6hPvSWm-63b9}|1Fz2K;p%xOQC?#T_wM~_ z`oMH0>ud-`>0hsK#-T@Oy`YRIqSud0^~=#ps2Mi}&tZGk9InUqNVp^3POaTKSf`aP z{85U8x{sf+vEeApJN3&_OF^3Za_=Q@Uo-~#1***WjTQ*pwG=d_T?U&ae7wLq0xy?^ z;M8Da@^}7kvTy2ihH{_sFE?p&N4+|EVhaXH)RQKVKQ$j)PE(vlKjDgJdd!lcxwLwm z$GU=^6H!-n)YH5H%e7$=g&vMVgu%QPbP8xZ3Z5KQs5sf z1d>Lt!05^Z?2*&wG%L5UI|~mutSG?T(~x9#ej4HD8L80XNvcef(G!kWbPG*?84gLj zL^!wU9tPOoCqtIYxKV`%>B3u2sZ`cO;<&bt`e^(l^D8DX)L)lz_qd4dxxr}ktqJOz z#PG#53CxMnVhmP)05e4%;S_zPv%ewMsGVWEkC#YY;3GmHy35YoU^3y#Z7m33nf?4&9vWXwr<$&YaPPTQk~~DYPxfcPy!^|kb}AgC9W!D4_;UQb;3R1cXI*`_rlI5avpBwelr)+f z;{jI_3oj|OG<0Il~Iv3nzC#@%b9uJ1q=v zMd^XoreRn#Ta3Gl?Yh{$&47jzj%3ZrR{n*Eop@ABfr&bnPE#kFpjvS+_{A8}f%Q+o zW|<8}o%#yLn#}1v-w=AD{V2S-5Xe!^yNTobkK@nA-Pm(4mhd7cXjzH@{8lK%@9m4R z)XEPBj+w*S`t2yo&PswZx5HAcDIjBciJja;py-)wl-sU}ZiNc;UKPMAz}thiE!&yNLJE|M#Sw7L6P z2CNKxoOzWz8@@&sZO>tS4Q)^?9u1#%&Oz?K7}lvFi`(8`HRxUahoHn5pet2_gC@3fCZ-W(^g zI+3uW-5y)Adho-X-MHv`9QjiD4&J*5@^g=-aDhyIV2t`K2SjUsC#>IlneZ0aU|EO^v0!_zS0sLrkcVfm|B4~nvmHY- zw>GTr@W$}M9_qX9FnPp#fC8Br#ANp%J?d`;=Va{I9)&Js_l3Zk;ZBnGb|KEz3?>3+ zvMiU)Vcly@$@HU6Bfs_RYqHDW4^;}1f<|R^PU^E1wg*~_0&6Sk7mS{#KmEessLXNt zbRq#gWJ5r~J_`i|>}$6)=3}5tE*@5k00|Fc5cpJYDK9Akr?*+6)}3WAki>=8gURG# zdMN(P6d*c_uaNz>ZctVKwIIJImminOMUe?t2$q(I)J^&*;V_SS8Kz_K$Zay5y_6h3 zcZd2$vw1}SNA#IeBsR~og2uIaoOn+jjWsO>znkAl^Jp9_Qz{@so#*l24=ensF&FX$ z9pJ2r7DkWVAhV|zgL8uk`bS35&g=`Y(JYM3)3y?$$veJ{Cue)7Q}} zLBKG>BKWfw*>Zt3VN@+fE$&~=-WdfTBH2Yf*7foS)?Ea-=PmpM7e}I;>5nQ((a7A? z!ve31xIe6ixE_e2%x3m;`s@dDociD_%h*&h(Bb?&X8|`G9&!qWcESpXA^xLLIMjUG za?h$Tlnb4Jo2y@v;U7-e86APkdw-MG+h*iKjWDhpn}r9QRM6Sq0PZ9RqD6o-)u~=b zCm)LE6kNDM^Zi;uMLG^$IeTDVhZ>mA*bBO+Yw^vrO%UyS9K~YWNr+w?jkG#IBE9dy zvd@&`d2fK`QpIESA$NaOk?@8L+n2D*3j4~Z*$%=a4^q`@p7H{FXvJliax-ggNE-8lm0 zjkT~|tsX>M{t>D3vuJT=IlMMAN7IeNBzD^svUK-!aMCs8yI)yKes0dj{b_~3eKkZU zudSkn%lvVB@(9gse*wJ?*J&U5!(aIzo6O7^;jb{yg!G&#prevWKUmMjl)f?=@LmVs zEtMy`%qPQG_bICPa1GwSZw!h~V%Y%_CE5YkV09i5RHceByM@P>F&MU#hiL55Ui#z z=c>@~rXo;Ih{2d8!f^0#F-Z#3#`g|)IX0WeEdPC;#d>-^kOLP^qS8NgD9{ZChn}sF z?vO;5vd$psg#|*?>^acb)mE5p6^W-e z#Bn0n%&-5Q671kJVDfDy$16(^f>z#!%2JB|_1J;-YPJ`vk_2UEy)aNb7IP(TD0k}& zLp+}}N+j|qZds8Ier<)M$#DisPM(Qf6Vmj}j(jrx06XXWt;8+NkmY{26oy>wi!|^4 zGa76>fD#3_VBZ!?ynmt*O_Jofe+HFlU70O*eayxMzF(mIMLo?w;>`XQZ^kWN_c2M? z6AZOa)6IH%Fja)*u{jdZR!v8poKoZp8`5q+U65Co;C1|{Al?V1ATakG@lAUOEe8eA zz~wS|p2dfAhpUNA=%{7?6-N-6R7_gV2SMEpC6xNyz|LGU*}d>Te$7Dvn*558rqv$M zW1mC5ee^_yu+Qk7*oKzRe{ifa-oV=U0xGn~o(8?Dq~2#XVvztB-Dj?X+#{P~J_^?SGUwqZabR!-yYU#85zV9*_65onBOHE+A**sK;k%N@(H(>EX zj@U{{GU*Ep;Q976GNqT&eT%Gs>o*fG*@c5bx;}(`p2CT&G=Likg+%@2KKMK>4iEpb zrAfYbFfV!pQnNpj@S){Ic1bY!v~DDB2a-`sz8?*yW)N$08{FEj$uj9m!Ba7mmyvdW zW4W`ESe!J#21bmV-24$3xnQ)Fo5939tpkzNP%0bejz=BX?`|~fxpdtG`n|vD>}RR? zt7pL?WlMLyeb!5`_p=2sR%04A`%{~fm(Zj|kgN4>GTf+l=NBd>(*5emRPE{#nEp8r z<{7Sp@C(xPys!g`?VO8`ezJVlaxM61ppO=h?711eLu5~@3hO1ehFF{TWYOj4_(sAO zDnr+u_!U$4PPDCz@ev$*bK1< zU69@cPkO3AK`5E;x3L~Bn+($4x}}hQPJ_9<)sE;M)uvNdDWDNAlIU7`k((dqk?RBP zkXCU5)Hy#aeD*Xtkl-F{UM<$bA&wW_a zdYewlb%F9f6*#9Nj2^T%<8RXLn(sVwO|GWb+Z-xtweW@f_ zNtxgmXTjZ7ssuI;8(BxnB=B7<&0JNn#)5a#=$Iet#IoPa4U3Jk)IH3>sJ}ckcVc-z zK5EQ>Q!|u*6{N*0>#+1(5pq9nC+q7gVWGw%mT3|TE58M!jhPzWeVm5=|F)3L*4y!i zRxLzZJ5m{4A>2HFCujNLWPE!p9S$fNf^MfO?0x3KoGDg-?=jl=VXFp4n25ty_MYLX zf0mx}>fpFF&S4t}>p<|I80Vpn9qGN{&NAB#`3bH~Fq9@tX2lv4=a3HCpOT8Z{HDPN z&qcWMdKH!DC}O+IO_aB5q^rWaA$&s~c=)e_w-b-CrBWK!ry79i<~+`U$3CD|-ol=n z1@wY+Bbfd>%SjdPp}ksRB=DImBfq_fyq4Bw`2BCtEqoYv+}{ZN{i2{dvKY;^!s+MO z{kWCC1+zbg!FJZksk=1@Pj8ALca2kMxZxP-J@>2#(fyP8s6okks$qXlDTDq%^nz`g3eN~H_64SnLgYD;azyROB@_O zh|;Jx_)XR)s{r+k$Ma9)UT8Ltjf*ZGgESiPGDJ1@X@e>}0| zf*g(=PC%bhCFF`2vCnus2uA+Fl%JwZ%bOV(U_Oa1o->o0tLE@lyfy307x(MgZ= zc+fSr$?(@c5WmQZbGI?p%t}*9j5*msev7rUXKpCISvD1PN99o}ehdOuh+y^%3tYM6 z71_KvlXW_r1yR8~bef@zfq6r?y?PA4dNq@Zmma7RrpcXks~Z2=B@le=>dvITih}0uY@9FTfjs#|_$gVP z+&?svu{1nG`nC?z`~U8fzUP`Sbu1A4q(xAV<=9MErs2DV3!$u2nBlDs$Nc1Ra@iw+ z#PkEoZ1{(IdaO5U`S|L}=oA z+JP>J_R+&{3~M<5X-%!4iQz)&rQ}fjX-3G?^}0eF)?BOu;q}cAmLVgt4&{`LWRx4U~6tT>5jXA zpDYTo@glod$%}uY?a@_ELi@qjVFe2Drxh%%2rCIQ}|}4wjdrhM66Fd$ynP zb$@|AVYUg)Hd@{c3v4X(4w*XbXtt zA(X9(1ZBBYl;7LN=49fb(cT5Nl&TT$@9G$CEeUHBAHXonx2`+C7qUvtnEwp_k`D($ z;mkh^COb9=r1{s;^RNfAJJ%YQUk6-r@+DlHvjJ4)X413DrXW>%1{xfkP$ca;c!xVF7-sa=^8{Vno$5fT)rf_WVr5Vc&Y{dApN}wnW0&uiH@O zk{bjclEL!d*(f^F23ut3GvDGpX!#Q!e4VEUn--RkpDQ%s{0^3hG$$K18x%2WhbRu$ zX%gePlejB~vK2O zS$;&|A3CY)13o_Rfy|VaW7!^ip#Bb<;a(X)zn+X@e8OUQi}USaWa&+M>)=rc-uDzE zm)+pJQjp`W2(iJh6`D-N>qZiODi%cy@>suNB$>OX6Kd~mBzJPMprrFEeR@ZT8T!eE zI~me=z%L54%7aOSbQ7IIXM%pfEU4iY(Z?5`z?wTUXs6`?scxd&h-O2~R;b5WYv(er zS`wfSp27@SbEf9r|PUH{DGJmE?qUGCu8XMdWYcq@CW^W*zj#k0V)YOMEMpd%l81);3cZ7lP@vY=~@?O}ZejWWEC^5NbU!cLe5_q5jNT5LI5NG-7 zx70`d7hmFiHb-T<4GqfsN$QguvF&6VT=}?|H2j^4ogs#Fp&bK$IE7umOsy_O5|oxYnB7#R&q9#K1a1+dr zZ3{f#D8x+bF^0$esrY2spyJ*k=$va!jBTdDL%W^)(P0^`wcT6N zcf|`L%$CF27SJWvYz4^PR8Q@;CtRreDf>>U(lmucDeu) zqJ4=zDP9Eum*PlA(N~oG^^5Qi?q=_zndpBh9)+43A#G5a5iAdcWry4#&)Ap9xgLYY zW!f}E?J~{OI*<3(Y-T%=-gJi8SNt8VfLykxm0CDKi|$<@G7-Xz&Z|T=Z%__4JB&c1 zauc<9&&SWT*Wvx@c%t8O3Z*@GB<1H$YStsj%*wRE`Rf|dHMkGq~BL&Gb%cSvAukjQP{W@*UvIz)+iJax*!uyEUYId|EAP1y81T zdv0vE%}}qiPFg*2EBqI}6%InZ&d#2^ubxVD(i|*c|i< zC7vd5Ui8G``2{}wXXh?~@dXod((M*j?+(G8DQ%!W+k))oHSw)_H<7J*g4`L|N;t%M z2^A4C%-K>c_;$6COx9_nA!|F(r}!H_b2I`ooqN!9b03t?*#_Nvr(3@5I*HF!9Ox9| z5mY^tf}bj5;fvcSxGLP>9JrIhTkklDyFg?QDv7tDWXfcmV{-xax|@Sr_chB+Ir`l1 z^%t=%sTf#@C;T++$9nB($dk>2merHEW7eT?bIcqSHx+{Q{fl(g0TJ*qS7nYKJc04^ z?_-nNd??9nqZ*7KH28&~!R0gH&x^B=+iZlV_&l@y6FUf6t z8N%7Yy0^8KN|DnavfxhiQHU>2!_I*=lKsRAYm#({n)5eK@UPt@#-+>o^a=IOqK#P4Hu?X!JTS z=pJKTj@Nu~`_wJi`))b(S5M`pM6N^0kA=`rHb1{6@uh9L7I`pS|6o;zDu z2I)%f->T`To!&_TJXYhG9Sbn~?oxacu>>_Tmw@`Ra9F;GLDCzD4~^D=d&O*c`8Nf_ z{fgj77)uCE&&8qeVSd%uX>fL?5uAGKi!FW0nGTdCQ;EE`U($O{fgDfqjzE z*fkP}i*`uC_^G?ZQkC`O2xOtdyCQhNn+cu^FVTkWJ7Bd{G*6*B5VyH^k;Wa)MB?aH zTpiYLDIRVNv2_Ty=iSD~f=|I%T#cMkuO#UQPb1IB2QSXKiQPTVI8|pS$fRR>D4Jji zE@QViJwXLH9PLRf1$m@+6#2wn$AGsR55?KD>%)7*5I0oVy&%&j`I>^^dCz(SF zF;eCo1hh&aaY@3i?m)PBqX3+OmSfxYF;3LD9LFcBm+CLFfn66GsN<$%FmkdBc&h5WtBI_M2Vdw8l z_f^2THX$^#j>fjXA#k#^2D^PFfHlE!*W_P^Io2NZ&{aOHkhg`Gn%7AGIw7X2Ck@|x zb|-t@*MY6-Y#92K4F;Ca@JK@i91KmzxI9b3o;eV94*hNS>6(6C(QY z0X-<>fbN8=-L z0O=3{~L83Hce%D?9F*-*QeATh zz2E->?#=};wdOiG{Zk+6>_Tu;#sU<4o(-;Z({P;-+j0B(3;Ud+ajbcWWQrw`QN>E^ z{WYBqYSp5-Y6W$FGQbVB@|lHi#U@$dZVM>Cu(5&m>yKP#8FZBfYqT@}d$KR7U@g3M5oex3R%CNhI?Uuy6 zAlq5Cwe2x!_`S0p&s&Z`kaz&JW$h(Jf8)UMuP2q+{hDIU9K8Lf3jVlj;=sy_Fk^gz z>`m+@Cu=T&*VsCAj(dwCbvBsv$PAi#jhSxVF6bye%$qyx1`Y|i7(eWWyU%XG+LyC& zaZoF~ON~Y6H}An}r8AoBN=4q%7??6!ooKD;CSzj-=ygK_&6Tt1{`0jsUGN2e^VM54 zeDyOjFiP?HA|Y6t^&j2Bf6Y0vCUx&*H}mu-xq#I}s%~Xj@W57T?N7Rg3H7#Pv;hEB_G+ zJo3d*-b>mMR>7aN;{Zh21Vi$c8a6K+K$i8%g4d7tWWdRX49W@<)xhN-lvYg+D=5LZ z=O9tK#O};8{u1ftX!27^lIeQT!9S*Yk{)YgGmq=tsCJ1Thw8rPn~6u`O_woFfQSTb z%`(8L;kV%aW@8ZE&)^@2Xgp@B4*eg_&?ZuYVbuaKZ)qdQYbxTJ(Kj3=oiy6K-4`O1 zeu9C12G4T~>$mk|dmokx@a%>)+;OP@sGo)ZDSg2^4VTD$BXQ22sgoi0Zy=`Ho3Zcv zvH0G}o}{czg;%o2@m#es)P=8vsAGw|GSvxsdLz5L%bt&;9clRK9^0+ey34X+>M*Z0 z0rXrZ$hX}T$_kFq;dugh`0Znszbc19H8S9GXboQN61Z;WX`c4<1F%~6AG~uG zLm_B~l~xhNW2zm>=lEbp)O?7RIm0*fngdt8cHu)$c@XF-g5P`Q^Kyq2Ih&M?$-9k9 zQF+&FOfOMHuk}edvY-KGJxC)X4=>=qlZ^yo6;&xRUQWE#%JpJ8)%l9N+)% zGEjJu1J-#awADHk|5KKtO6+IeSnwW~X{O^oW)-}Wmx5&)w@Aw3Ak+vs2T~sMA+aZc z{@B+`&qgWW?O95oKfV_?=HFny0tVoT0`PhIPJDN96hpe!uh%9d7tBte zMN1CDCq{+4M_~-D*6ZW%OS3T2SRN&MY#^Xf0=5}lp*+H7w$@3(_SHM-$_8;*gd*(j z`5Srn@;7qYvT>Gy3|C!3ov{uNhVp~u*!glkO*%x$*@p%A-yD5- zJ8(wN;`$lc5YFx(=NB%cu~9=L^pFNh$DbzuP0A;a;v&(+UL5v5uf#p`xcqG?FL1mo z2InYj0_*d~;BUz^wrAamzlBm!CG;I=zX*ieIVm`6N+#c_>z5@;y7vE9XQk$h7X5B zQA^sD4n5O=eCoxYW$MTXKTM-ro!ZFpw9{1cZX(u7>hNQ800v*xpv`U>JbpKUY`fM# zN;LHGmDU{`PW;UF1-8K5qj{k28iDmW2T10KFT^*slANMku-kZoGz2$rN~kVlc_fky zy{;jv*i0>F#fXI|+j+=|`2c$yr150?BB+QyY&m}7Kg++F>#*tXAM`1^g;AS!a7(Ae z(qk32bOY-`2-(m{btg8$vY})!?Ky{;zQTA_REt|~T1ZR^JW;9m941SOp;1!~=NOxB z8Bx{68?~%&ecf{K`x}X$t(w5?s5i=`&cej-yHs()0)>aK&|k*;`JX2o=!xg*cx`_G z|HGW0ydV*0xZcg9dGcJ;Dr4DHHM+3>n>8Bfi{r7Dr9_HfhMT>&q6n4ajw&99aP>%V zTiy$WkHTTfe=cx)pCJBRnJE7h8He~N1_{2F3?YAtz=+KAg9KV8J%RPh=!)3f7wrf5zS%h*I zE~RokPe{#CKHb>;ot$Vog6COw;pCoHIQ5%iVA>$Y_vBpT?@pjbI@y zz_5X8_%7H6lFge?H^-9-e?0};##7+6DC=miI|0Y7=V9Tq)1YH<5eBWoV4koZ3$3%{ zP81o!rv<5Gmgan3wwMJT=^EuMoGAzGBnNg+If;K~%5hyb=)#hf6P!6Z|8U16meak8 z&`Y9juxC#XneqHMPVN55OJwIhHe1!W{t6O|*s1e4Mezmwo^lh;Jvoi@)-X6Zx}N5Z zW&!g_iudy|4}4v^=rOw_)Oi&R_pYqKPir*J% zw4J<++p8XuZ`Gm<)tJmR$cv}RT^;zcTa>AZtVBK-QRDR*Tz@?yrqS>a-m$m~6K6_T zHO3^SMO26DotsWwmJEY`;ceU)AOG!2gk+COB6w#K zoC}iy`_me*rz#O*Ja&>4jYm+fJAf|lX5p-dUf>=k4F2BRFn-%|7#uu~(c9Skf~Yt4 zv|ncTeh*-9%1baj{1oo}90j2#T4?cj*U}@O1z`3bPe?m*8H8JYlbKO{c+Sg#`JSRrWbaX^ow9;?>7>SG z7Dp4qkuo^;>jL=9bz(S8F+^tSJmQ{z3w2^lF~sd24HhVXL#g5%X?SVQaopor2HneFLyUkFv;Rsh$|h;Srp|7BJ@^dNo3~>4k^MMJ zdMioU)D1?a$ML2@EEG8#({-BlFfw~Cnh*S;Z-(z6nb!<;vH`$*8-zs@$tdJ{f?m^+ z){I}vMNPnh+l2d@}DI&t89ST>6Nt8_9K7t0eSwJ zcS^WRXA^&2(*#|W|As7Ta|K(-fu5v9y5rn=G@rkRbx(_OM|c0g_VouaRM-Zh2W8-l z!5M0Qkj)zu_;Z&066DHnQ{#5lv5XVHmjGj7#l}iuQbbjPM(UjUs$dT+ljIk3gzFuqKc1R z%!f-h$MM7be2{XT2Um~(p}#^G!u`Pk&Ids;eCJ$DEsW0MYVmWxds;!Kd3qwBy9Az1 z|IPE6Y=%Np$9d|;@gVAZ4XR#MpwH(v`q@%~-*#vtf8FzH^1H7cN*-UJk7D1FkC*(w zyw?KLlg2;*Zje_256R=fr`Y6Bh9Sp7(7)z3=RYq4SogRct$)_hf?cAR{cIV1;dY;s z%Q6>gQex0XZ$3@@HpuQkHiK!f1n%*hM#5f3fR#!HdGTWr8cW22_@&8wdy8oFv)ln$ zQ|{7A{a(KC!bU7m-4DvE&qAv3DsuU)9vOLdmTbPi1$gJbk-)EF=rlVY;v(Z9v0*Jv zzjB3K3&|(E(hRz3)pm$HIUg64L_pAcHowu3fCXFLL-#Di94{Lvxci(8H~i#l>>1&2 zp_(wSN&t4b9;FS1EN3Ld2(zNXkgLf~A1vE3p3`l4bzUM}zUDIL!R&1;H{c-d?pR2c zJXb+0_j!2nK{CE5^2Q4@SWcEPJ6~+N3_s59N4c)={0V2)MfIeNy0UY*Wfj#NyX7y* zk!R&pryvkq%P!#CbER~et~d_=7~q?gBS*|5-QrN}3Op(-f-c+Z$@0d7kT7`>4$t?) zgZmeP;ruo9O7LlvsT9K~=O)s~6TEXUJH>3)X|uf(Gj`5Isq=CkY|V;-s=@tDX8h&R}`2!cdSO$_ri0GI>920XvTtMeSf<+=q4*iTHiZBRKl#AJNa;iq*5nsCd>Na@fr)lfC*9&~ht(y{f9psE;#(^!t=uGg~Aof`>na_+%> z`WU>0mjQ@0@>KY7a62jp1^OP7sXw^vti%n>WzL}7&ux$!G-0`HB+9~C%m!ms2%OLDyAO>3>hs(GX<~Z(yiqGdkd(Q#9&h+v}tXk-R75n)o-hpHJx1dD+Fm3m{ z4r7BJeCq^B{^WLX5Ox>immdlt;}1uPmcKEYIkr=&QAO?xLv?s$FXO*@Gw&6k5P$MZ-}vVWg+H}CjBK3RWi4Bz#1Sl&Jpg^fS`@pYsd zkh99zt@7u{1dom*{_i`Mq0{JZMe^hVVT(kvH03^1CpxeKK(8ic*bqH5eY(r#o~CKrlRl{BVpwds!8nR#rk1{Efl*aTaVZje&BO z^X)&{L4pc>P;<2!eNk%;k5w0e$=0v1WV#k69y3J2(k9f(%m6_-Hx$fQ#q|mDn6V0IWptQ_QQwA)Bt`U`WP(;k=}SPK_auVeck4$Ex11Fhy+L}Q5$eji^$)5;g4 z#fI1X^t3hnO7B5T9Gk+J`LlVy>9b&wbskjB4xk#7dcb+zO}zF)fVrQn#@$%83RYb0 zz+IO^EH^u8G2i;1a86x*54Ot2uw%w~RB3&}diR8pTb)RxROf(Hl^EC>KY)p8>P%ba zRm|Khg?+<@M1om`)3|#u{e={_VpcmO+Kkcw+jtbozW{N{%GBp)2X6C;B(iC3STv>$ zSu-c#h*1hSiHUG``vPtn>ZPSlzwp?qqeSwlKG&T+U(Y-IAz@C^I9E`XaaFnpc5*jy zYo8hSa=SJhKVCt;ml)EOUX{?&*@Ma*!d!j%k6>}826udkM9G0>vS`tJezl_v$nO6L z|AO9el;38dLdioMzsa(RJmbm1L!lNs9j~Dyv4Y1PF;KaF7rlEX8V(-~hJYjG_=;D9 z0)HFu;N5%V7+;Qh$|j=R235xPW-WY+^QMwz!T2}mE38TXMeqA7Q8CdXSTRt~&qxr* zE`z-|>9-rc5Dy^*7XK0Fa1Rg=LAh*kaA*6Z1!~|J5U0Gid`{ znusH7CnWI~$B{&4DPadMly=TR@W(-{+i{84Cwvucu4UubYhTEB(|d3@F9GiCu_da9 zw3+K~%q(<2$kET?+Muj10>67DFy^m396rQ>M;Gd_a%mv6><|RO9xD_vt3wxpdT!u&I|jqm-D&!wk2Fv0Ivw|YN=J<6Lb_i$UNWiY z-@aJ|{5feLyOZrvtIT5l= z$pbdj{PAHr=go6|!sHzMq+E&ICYIM8%ZJGVfzbW(1g0;Ep_SJ!fmW_2zUoP$#z8D2 zrcs%(tc;-R7ffd71Rr3-u`PIuc}xRsezCLfRE)hg1g7sF($}*)(9-lDd^_}j#(K`f z{ecJI%;8J0$#*ZzG43WocefIQPa(KPe=fbi`G_}mE`yLG_M98iSyUi16`R$jpul_| zNPIia`Q`f-nbWsuuj*D3#JWse3h%%*#iJzo+!HwQ$PNwqWvhYR%;;Mx=DC)R0WG0^Kq?! zG}`x!@vl8;<<*_fh5LciVAc12Y$0=)owduWg2$)o04?Z z9a!@#7zWsU)V$3B(4;PbzSo9n!s&6k^D`G0s?=f3al&=|DNSxRiGtf>MQ|1xrKeAi zV!8Qs$SC2U=i71gd!5DZq>$r#n`L)K)N?}rUBzAfh^t_bypWK9xi81~SEC-HaMfO# zV7&)wzAeT789EPtEZZoKQ^|@bMMR_&B`Fo|>pVtgLsFzgiXa=YM_+ENTztOcUZ&G1w@2rs>tq6$0B z5oPL$gvDDdlq?|6%4eh4{X=lv=O!`tHHOj(8Fn+9g69u%y_EPvaEHh69&&x=-1IJJ zY)S*=mNDEfe1RNDG^eIqUU%}GYB-*$jK|I`1j$o{w49q$&(=_q>AwzV%ud67SsD04 zPM5qkJPVTfT&6VW7#6JZM0dSf)O#Bb+gIeFLX-h+3^)nmX_Jw}YojHz3&UPZvGd(a z@vMm#6&@T!`+gh>qgD8>KQa&ts|J|6ab1x;($pdc(5 zS8tuq>l?a^_j#*1k6R`>)w$!wx69d~KV0uSX$styFQ(rVe;X=1*XDHTrJ#5?g9x-8 z!D?*>Z4!Ts3U_znu4W$oe#~Ov%C}G%9Sy>lJaCfAH1g2M{MOScO%Qd$3s)W<4@x>` zS>3%6IQDrV>m9yVU{-w|=HUZc%lgBxnhqV2`i31pUx1&!EVy(;g2zk>nUd$xu8J^l zlL6I7UogaG32xgXj?T%aIVOoT_$t1l%RaQyw{{cQ%Nrh|k&!KV$NA0c5+r$_;{s@8 z@CBG=7DhY83k7nq{;>8_IDG2)Oy&lhh2z#lE)x% zhX`vf)yJ{hR>R|ikx=|FlKk1VjLlR#i$<}@#O2loEKW_LDVb8JES?ELw~9!nbs=rP zTa2oc{E3KC44<2BN#5-`obkmQZ`7&^+7o<8XH5;%?u&+)$C~)NyqXvt@4-T7gS4Kt z(2{%tRMY=}|NAMZyYo53I6T22E~n?6zMSq*FYJNCyAAM~TaNS&S^TBYT%GWuBh1$LZUI zP+HHF9oV;mWSR3=$AJv0Jvo}0`Q;eY6%SM6n~TBq;CR;FaU;Z~RKNnUGLZYW81K3| z;@FHT+J1i?Pd%|6ZT?$=mT}&&X5)NJcoU13f%({T^9j?f^@h&-5JIo4x5Z-lc*eYV z6jqt5BcAIJtT&E_lc8O>P1X|>?}veO=1Y*ORpNo3&EWR#?frozB$BUm;e3%4e?!_2j zWOM{GE&W_4vEwWZZ%qQV=kuX=#!-P%i9EU$4e>qQwxar<7})ya2F#frk3WwKv!x4v zFy2Fpz|inAG${wM;@2jz!5ULo>x6fLC)aOdNnHSGwEjvx6wF9zSQ+O&Y2bRPW^kxV z3mrX<(Y6)RU}P!5dp}7YdWS^V+M^Wx*#?ko`9#}YIsU0pB&MItCX$oH+3m;Q3TBC_ z(uOCKIR5xHV#?i`wz%)csNhAUR5_H&p45S7i+98DR}K!fQ-$Ya7z+(06L@zvCPEWV zVa<<;a(;GmcJ>Q-Y^pOMd0$OA-cAIORqP=vKOJHD+wNnX*;%UYMd+2Ik74G|9q@jw zf*|2x8a(;v2sa7_>BIXa^rAhv0UzkGLF~XQctRO z{=|xvA&}^oOfn=&!61+LQ;;mt#rPD>xy~I~v*(9>a^}0fKj&f3(Kq z3AW!Jj~n*g##Yy6ytTT8v_8wB@At~^ob;XGyZr$g=j=jNo{MAal})s2!wEQYIR~DV ztFWt{_)ysXiWZ6XQ1{$Wj#(4WZmUY+-ck(IDjwC*^y<<9E*I@ zCBc5L60{voMSVO?ex01ijz8fAJ;fGKd4Yk}^b}Yvbrver#CUHT`$$|s0mpRCBLk_H zblr^43$EWKldi(vni0&<2bwTxs~S7uzA9Q~$r2H>ZbtE4 zE*0PI54|l)aPUGUH1h@Eu=*UVY*HdY%Kn&S_YQ~a%|M87`Ci5gJ9?hc%q#;ed+;CZ z@>)zog{^7eLk%*$N*(Cwb8uo`G=80>$kXZi1a<{qasHVotZwzjg$AFhh7Q!zylF3x zd<~&vk@0l<(ImXm@rJsN`-~Y&VrZRx2R*lVDfu-ro!9Sl7Mu02z|h9KlsUrPp=%x! zzP}18^@_5FU81b2k2em@O~l7bI$%)$DC@&JgI~WD;iQC4!JRXXI6*@M<@PsF)rTU; z`>~((u_=ZlL8Wj${|AjX8-OZ34rE-4>+TRZJ29P(n8$%<*-LDh_6ain zDnP4&Kc0DwP{+!G^73Bf>-7Ralof2{%kpK>22*cM5ax(Bo8$ zjvvQ+sgeMLNuAWTaRFOlHN>C#Ap$;S+u^BK+u7=`hr#b-H~MB}3lu`GL+|Q-`tavL z>{kguCGcnWReC}#h{4fSDfrWcFZi_H7~M@Pp)N**_eFB5;M)Rjrr3W1E=-kxIJxt* z{$mhIgoMEFGnE)#&Y+X@DTufs%6caKz`(PoaO2bqus&=K7I9s*o17mb>Ru((N-N`e zkzPm`sG+@s8mx0^5s920gmJ3t(OKy&u24Tn3p|8*D@}@M%fvLy<#I#ycTL$d6Zdh> zg3Flsa*RBU;hep(H_+~$6V85}Km|6w(7D?jeWV*u_eTK=+vnhMKF7>x9-^g74B5+y z6Nsi}4zwl8^JcF91Z8FU=&tTcGcN_h;KEQUQfG)wJXOelcN&V@r?7h^u7R`ARCeXA ze;}(Q#&eyjT-DX7L^OvIz%)V`3d486x1T+9hGH4E{<`$%kxh=sZ1OIOUB$d-lsMMk;$6$KB{-?rVG^ zuO9|;GXdgo-vtcf*P))T5cUeI(DB#Pi2oW_!Gk+CsMf?nT1hvRKmQ*Mml=iV3yY!j z(IuQH(5In0klxPmAhXv#AWuK9B6gp>;g8BT!BXEjZ1yDXcgaX0z^jMk=B9C^!Eh~@ zjB4>ZPwL}nv<+R?n1Q?1OK`&FdggoGU1&M_m;QaO!lp`cnQ0Yc7@2XA>7#nkR~-T2 z!u5FA@3!Hgeo?kTL7gtYyPtNumeX4InXKKZ5_){nVxFUu9|6&9d}i)%V6B!z`^5hd znC}4N*Y@BI=Cy#aZUrtbO>7hD;mPvFaCg%aC{gCvM;l*JC-+R8VBP>_r+P@DNIfpG zcnZ^cC$a@RLvY=q3V*i0Kt=0+@aM{9h%#8p`#>Y{kF5~v8g)~!`;Z0<8PEvBmhib^!#&o?O?+SCYlvN?~= zmkf@P$8oa8^-#m%f8=RW0K7f+n@}}Xh>cVR&%QgvOmqSO8UblsWrehOQ8Rbztrwl8hCAgLjGM$2MhU$Xp#I9 z@12OFi3j9Js7e@p_oV;}l2Rc-v5nlfG6pBvGPuCM&s5*@;5Xbd#g9Wvu}Eb$d}-YZ z_Tuj-4Q~U}4s~`TeGZ~0+^CV7Gy^ftw;rs!W8@O+|NHnC)&BLxq zzmScMh4pi}Ijq77pLPw=s5jp5EcP||FX1A*ap2e^PyfKbIeWqAzA&#^vH%VZv=aX> ziBQyS59whKV9DD}Fw_1Eet!EDMvm!&_q;GXKKnYT7_+ofa}HfN$rD2_T_BrsACsxw z=c%REZCLSiJn!FyFXV~NsGw;|GLhR64Uv}0XcbySf`;dz!gV!RA~Q@j@A*w$CSQh2 zv2#E_JXa8177OMP6T#tSFYLC-6?lE;-fXw;YFvKBqMXruU=M}I#b^?j zU%6B$gG^B+nN8IY3T}a0Q!4g+4aO0{1&Drg5Yn0tK;&-jcm8EQ%4${7q_4?Pwuk#% zM8tUWJ7$8DmlAwB&Ekx@sW7dH&oR6uaP6*cTz2{wo=?kR%6MGfXM!48n;%biKhfak zhrQUGbOWRO^#~md1^)7I9Ow8M!as-Nd$k(Ac!xc#0konl*dDYZzA*sW|Gk+ zXS#@+jja8+w~-20!S*Q!U``^}rT&T*PoTKX3Ls$NCe69IUs;|z$u z$ftex3`q6eYJu;!@AQ);AA|bkcx!{)$=3Mo+z#;xx-_-1PDc||JwDNOOBRD)=rB3- z#R#(>PQ>(*dth7_46}mA@%EI;;D)S6G(Iwu`Wy-)t?PE8*%b+P_uPr>Fvndq+ri^Q$wv)*;KSEmCm1^g8ob2L%psV%ACH2^Djoik-RWSmQv&?AM0n( z^%ZffE`Y(SN{o(`DqJ}}7hnIpL+o6gaCg83+P_G^d$($_2y6^9F)k`R6b+U_@jB-!tk~vr%)gA7p+JCpP2H z!r-6*q?YX@akci$ioO6U`XrdlmOo8AWxHYja}jp8`&v+d7=gTa02Pyo!l0W%Jl>n* zuw`iwu}_=8-kj9Qz>zr6nulb!=5%;fR*St~e$%fd5-4uA1tQ*XU#F!5 z=Z$watVpIRYTgmb(>u7j{7(3V#i_0Xo!jzwRL~WE~*<4;t8be-C z4aZH;>TL#Mudk4qA;%$A`4%lw_QB61J%;61b*Oywd2&?YGrSh-CKGPl6Bw-0WOr)M zgHQ9go%*K%ZB2YZ7C06#=7nQqvl*YhzbJzXwml(=!?wh4RU1}&q~c(t4GQefgYjS& zj^CgGHbY~uOL+qvd2x*{`xHz3x+mlOxdcr2%%#hJUndLOatw#JPk~og&*M*JJG#Ky zpSb-thb`Ru^TRDMxcg`t{2o*nFgt~ydERQ=xPIPvy$>6kI8J$1G(61E zzzcJKfNbt-JUYLXB*Z6SNBcLjrXmil5*ESZt)6i0YKWj|kv>t(&IfUu#qdt=7nW=( zHi+CXj(O6rjh!k6wCidzotVc^6Ng6pW_Fa!sHzYsNkqbS1yNYl=mhy4CAb+kf*W^V z`5YY%KBX^cRM0^v{4R-7C-0E#m`wPy#|3ZR7iUzj`O$)@OW4?iWMXeu1^pigh!{tp z*85WG()p8&1?pnVgL7bUYnx#F!hcYBKoR){N;F7)8J>#e*!){}k*GVZ4W_T1V@ zdHfJCy*dJyH#PzTDD3t#{q60R3)P# z%g|*`5DwLJ(5~sS?AKOL*eD)BO8(2D6Qp%f=*|Q*KR*g7Z#Wkl>j=T2qR2n}n;)|y z8Ev`!`T0l+=P4;=teryz2j=ACT#Y(>YVwi1&+UTN4KZXTm+O7Db3P6{se})nd~$Or z9E|E$;8vf_AoJK7dGEAPcIy>#+Q1m*Pnp7ed43Xa{aJ;UX9mdrNfNj(O^Rpaw~)5F zWrL)31v8;(KlYdpQNMOwklwopG%w6%)0=EDFeHa8jy7S&>&C;_HU~_r?-QhqDznU- zxiD1FOFN&v$2Ir2L*%pz=zOu8yp%EIzQ2>m#A;pC>Q6&4(7bBV)YsvElM{)KN*fV7{eWt}RKsiCZ^%dP=kV7%VRT3o;ip!eCEvBhal~#lXzU%O zDF=+1-Xq*wj1xv8K-anm&i2{k4)<^f zwj*hQl1Cd=j$t0JhjEq8n7<{=us_x>5`-O?nKgiWh*}o`>|(tZedH zdjp>TX@UHq2rhqCN_V+;kSLFKyq;kLKmLe==AUzzmm5d!ey9bfe@OGOuMskh*OR z-Ju@{TWTEWjm&tA-{$~1Z;#_s#Z368dk+6hGr&ZpQLwHn0Uf{T^z4WP`?j>+(7@^m zl^j0IE`sg(PB1mmffb$0Wud=_LFLR-xZUhLIrO3gT^v4B_iZWk zqRdx_nrcGIw@idotIZ%Q@)Or)XQ1%QBd~ctk7ugbhCxNrF#G9C#(n-$^x30@pP$Zy zRk{nopw@zwmy*J1p^4BbQ&=cg44e9kNjvGpU}+w-RlMVvE>qdQ zLe6EHr2tF&%c;?cVEi(!j4!$AI&RL$CbmCTVMywJ_+BWW@)7Te*FZAeIKu!kl>gzO z3=`VVq~U7g)v!CS8Lv*w65I?==H3TIZr$WH&_(Kmq(5E{xXpsePn84p6JKfG3>9{3 z#S*-JH5mWfz7R*PWw2u_>$V6(?@YPaq;vv5KBjuIk_z% z#l(NxD2+duLu|~fc-HL)Q1fUb-d--jyE9#t731cbKLdOw=0hmNIvZhf=}AGEi4h9< zSmLq$?@%&d4z0SDkoOd1D2{AGG>Xrb+4 zZbvAY!j^c5;PC1WkUy9LUqm`VC^;79K8?qtO83C@T`SuCG{Z6OO(Sfu5{lPa!{=9Z z=<^{O?c{6e#?oHU9;_n2%>prf&vD$@xd4u?*2I0BJBxSq5nf+#6Bc=#5N~)8TWeD{)91LNmLAMBKa+5{{>%+oEuMDesFf zw@gIYf_qT&sDa;oLmBHN2y4386#nMwg6OtkJkwzX8=7h$FnyTJ`0R?$_IZIG*Q;oa z<5Qy!IUWb5z*9Cd)T%KDD&x4DfbJx8KlFt9>}bU`|0IapmK;dA6Asc_{*hH3vOML7 zT<(H%NT>#NFqiAOCX|dpyExBR!EVrw)n!w@>(cy|aH^~v zf`9mdc)3y5SURo8bRi4Ou75odlyZRtvVJUREo6_9P0=ztwgnbszaIQ@!{COuv zRy|~qUJxd`I~CyM>*c)4EGbxa%?9iA0_p1sYhm*DS?E+~%-SB<1t*@`!p1=(zUsE^ zsBduqw|SVN*rJ=Td}t~@=CV}IU(VtDrH$yTa12)Gg`@pOJ(xa@>u48u(A*V%=qfV> zzIhvGr7COT@-1P;n{MXN!Lfh zc-Z11ogj6ZQ51bjjFqLiH|uk-USrDLhR+m~G z`6PJUdX0Kzmtn`HA>7Ar!K$b0k#{x>T<5Q#97~r@F&1S*v(@RFw@;Y7SNSMkXM%D* ze8%#a71A@hhHUe9@X1O{zMV*{*#1JLJwe2s*>vv*9n?e2~qo3p_uu#n9g3T z%v>rOAX$Bv@bZgsta`(JG9)}kO=oOEMVqUz#@hj2EQy3>?k1ga_&hz>EyPHrECxLV zQ#iO;1>>D-xPHKN_Qd8u!S=EaGWDc1&%R{>EB*RAo#m5^nHo1}jYgRu({g~jF(DdG z-U>qtLhx&?B`wb!0Q@v8cpv$S9x8uN`17`)UfX>*d4ZdKC!K{SEw2r)d|bftrSHRU z+3~#b?Y+40GnZj`FbWGEO#t4SYxwBYRiawI0CIa?!=LkS$@jRw)bMI3Y+4r&LXiXX zP-_A8SlNiLlS>KT_6W1$+f~r=@j;<&OUa|To@YR!}cle@!FN85#o%DqPp z74Aph{pnnveJ1=`v>(J(A5z^^5#EoKSo&^B9F{aEK$mDQjd14EkRJ=dXYWDsK2ZgI z7fhyyovzcR$34M8IRzfZ-p3-hrP!BI0!f?d1)nDGN0s4E|3RWF$hc&wI=w+RL`Y*N~^``j1vv)=$XVH3k zu<<ixe36+74JL=t;^ZT`YQ8W}t6PG$ zY(YVjaTtar)A0_4I?QwPY9ip@5I5t)7M5XycJfn9l)L20XPkj2r=uLKk3(Or_ zdc_jk7sr6xfCYx<&&J~~burpi({SIpkMOMSAOA#=C5U(HqviK#IIFIVJAYQ;%*;!$ z$Mqk5u;#C!fnqk%4>5z6Mmdl_(^p`*GYN$bUo;%N(@FW?qjBch0P1SCiE3JLo!X*I zIOC{`CiFaVCKToqe;ckWjUq1{^_l;cdy?Fme!9@+B-{)yfVt%@jG@RpP~GJW&A;m_f7m z7x2qA1)$}RiN%V35_!6o%(y*D46n_E9eLYe=j>H*`=lk9|17|7e^A3nF6ffBo}oz=aB{B^7EFwwjIIcyWts_#W^@aj&<;Pj>tR{J4aWUo9PHQP z9NH%9>9mnBW|Z4AM>J+q?r@JYANx~jM-6;)umB_5&cN*0tDv%C6Q)|IQiW5AXyg)& zH}W{fh*gs;5!Z61o>b`%^ouxALT(-pUk1(Z*smREpe>7D2?9+QsK_M3K0De3wIx{5nP>-h!xwz$qhG6 z@_YPDXnCQDVU-*^EmR4AsdG0`6*IWuJ`Qq!E`=$+%W$h>GB{;x(nX6)1VSf7h;d)0 zU|4-Ot`1*9KihJB&=fA4@I3&BmImU<+J5qI-zpd$pMpVicNp#}k%M(F{&Bq>V{EZ3 z#-hy&sf^ZPOk4kr&M_^4b%o0Ghx>FOEqnB@4F}+X+6mmvV>86J{~&j`9$$jV2ACPP z8Z9P^fKybJ;L!Kictybye%cq~PjhR0HHWq4KP0h#Z21=VMpK$+wM zB5RZg6aR!5erPp8{h&b@;m<;y6U*s@;VJMDwvlc@FERUEg`?^p>9DT@=vqlb>1U3w z&H0Ho1%=ZX326{f6@{5cba1TL7KEE$8g5v;h`i|;B6r@zkUv6AaPeH5;Gpw+;&@#S zdNiiv`D@Z(>vI+DCk{f%LvErybDHplg{XfR;_I}2O3z7R^@}J*qxmC=vz3NlEw>oI zV0~E2KLcir7Zl3tL)x$9xGGHxDmb>q&mI-%i}AtmKs&nO^A6x^wUMW)PpFng3hmQ5 z30V!rq^jpU-d$ish2D9=Dj)8qHo+TGcd&G=v>#r}{6>36yy&k54>^v66M9X*428av znXaG#62Df!7}wcVeX8A!4fPe|u|onjmY2c>F@(8&QB3~4x#Y;Nd}1f}fM`$P^IzPe zaC{u+IF*(G;T<`G`KHMzE4WUh)1H&ZfijRD7X)^@ztCWv8aiL76=jY%!Du$aaQtvu z(w9XT-yJXsG}$+aA?Q2n6BO9|fDX|U@L#|#FuxMVY||0rWkp?w&m|S$&-K|o+kfKU zmN}$O`zf)nR)=T{IiCL4e3)}hfEq7FQS6Yi!1JLL{;;rO|6PqkU3Vu4>^}gzBV);; zof#O^D2(k6>tRaBdsI^0K=Y$bu_3V&@@#`S2VXLFU+aeRqPlqNDc2Ry&Y?yZR^$E6 zQt&uz19%Ee!!hGXtOxrwc)E^KW^-Rb0k_B}8=1C;iW;O9WAj;z}awlzl=kd~= zHL!DVn8SP8r;@6oOw}Q|+tpT1V?BxHn5aF2z1e2(joOjeM z0rxzt$1lEzk%WE3sHcDUH)fiGPoO9SZkfU!QHv*WY+>eTY$cUW1Dr>3 z9dsU16O2`!$88&|nKVOgHjUxD>rZYnl95+PVoxf}Q(p{XE01tq$`Fk5cuX8@T&Sr* z2(7Ib$CIw=yhY~ahBI>C;4hOR)U4;aKHR(Q&c5mV`A_{wjZzc*hR~q%x5m!|R8DCD`hyB7?6)R}hh%u}% zo(t9Dt*9D50sdRD6;3|ej;23D0fppvXNZf#;8Al9@M3`HEron3<%FAKa!x z-I0yBR&5q)K5HZ@_vCTkCXS!sq{#%06v5NBnm$KVJ!-BmBVRJYwmUTvY4SMT?9$l01>YpyD@>Y9SA6IsU)& z-C!&@I19xre2LhjLg;liM9EW`kl&w<5~n-(r{TH;&b*o5}_{ zAA@b*mP6XcmsI|;51o}H&9VmXhuDrkhr=HW(Rsm5++=tHV+ZZzZaOefBGPEWCMbbH?hLBsQzyz?_R zV7+)hTz{Ji&Ksmz7w(R0)N>NHd3vIT?lu^nzYO?959O8tyK_FllgFA2chnFXGL`GB z&XmLLr#_IAFTc_2E21D$u|V*jlK~qneHHnwp)m6_=K_D`N!LHB0V9KT=yc_Q!Q=7D zn7C8NaNMLf=&!@wHA)kKopuW&uXFy3qBeT{%U^PB-elO?#m5Y{#qdHS6Zw(C>^r}W zaH{n_Ga74)3*$#%;q?fbvPT6@->inFuNycQ%NE?Xdmr!xR`7P=Fjj5cfr91;ENSC7 zV=gJ^=#x)Ans0)tEY5o>#Nf4+n=srW9iz2=!&8oRrtX&kO)Wl9cS|3;dOgWi?(<&_ zwQy4UI8bg#qB^`N*md(R9$nA%@ivtT+GIs>hEW#nI(8D4{R<;wf z@sVU1+V2P4p&boJOt{>Gln1tdIg6%i`*8obTX=eU6#R<~!kuyfRQ&V}P%QU^*S;*B z_4quVv3rGwasn_ReF65PKU~nVqy5u4@4(tn_-l3u-3`Y;%S93BmOA`Cu^5svDfSO4 z0p;B$v;LEZh8ZitOH&i4?;MYPU$>!RZy{{h?u4ezqoh4;0&$DkY-n&~21`H{b|(LZ ztAB-2qstRw^x|MtJD&dj&jAWFx5EtS{g^go5`AMIi8c1(_*72~90z*@d4DfMmqkNW zvg#IMdt8&f&3{6>>Q(5a`g?fW&Jl^50~6AdMN>wa$(i>V%&{BlAdp}wbGZ`?(>p-v zZDGfeKOY+vgK97&2#|aeIZKlO4tFT@p0eM}$>*fWtc}x2tRDAN(05=m__!I-F54P9zAqa* z#e~SX7h*UZcak1RF+_(inYi(99CBD<6loM;zlik-B0@*VGtoX!I&>W_KRQM?dQHOv zI@2*JVJ-C&yTVU2P=r>l8 z=7Y`QH8>Eg3VqkBaL*Zxz$ zSMsfYF8So@O-q_Oh?7eW)O&4*H3vC)Tvqpm#7OemsDmThIHSo3iFF^;>|RYgC7qa*7B$E_oJPes?aWW=qnMfAtV2UIJT& z|KTbw`y{xxi03IXvLlc+Ee<0^Ca_D?r?3MeU+D`& zDV)>!mvj0`z+E?MTC3pz?1Cpp~Ulr7KJK&*0j^p>{25y^I zN%?UpVEarBpQ&~W$`t3ZTb|9PT?(qaAnpxDzVa85-_nI$4O#ex&tp9vZ2;?xNaF67 z4C0xKA@$ZnQ1q#W(FNaNb@CFy(1&g^*Y6zXV4RDUX3^xEKKCZ5Cdz&-qHyfF4qddM zk{XK3z)#E5jH`J7qmU5}7fp3>d?B|(srNCTAIkEo=Ka9j4kb7er3`_ck&v2q8%jJL z<0P1jTCR6Vj|j&m%xuOSi^X7K9Rn$RJb(nCh-c0W?D{utNQAh$1JLP z&j^WLph=%n@xZ=v4uJfTsIg`ImwUdEzFQ%zpYT~+MK2;QQvp>215mvmM}@y|pPAJu zs$O*vpES3VyuV${*Iqq12NH0dY{w6Y@5u4oatK?$6W9E?iZ|Qm^8(LCA+y{~u*43j zLD&H(FV|%pr^>VO(}O`pR1;IDjAzpWRM^ow7s>ra9D6Og3&yT-Jo?8`M0E5jp1Yd? z$5RZ+VcGHQ(5`-ilS>xR^3*80%XA{!JZ%g7Yk2^*Ee_=Pt~P3@u7Xc4&1aboY4*&( z7_*eSi}eibC6jd0(3IP`4Yn|hMMfQXo?8r;D)vM0#)+tLY7V`un@m%;g;BB9)~xRb z&iOs=DvZDc-jcLlvh!pgPLycMcw>4|xR>4~UL9ANkJr@UoAX2ROn51z&Qrm9ogA7Hnoa_~M1hfP25uT>4Rvd? zDe1Q%o#Y)@+gzk~Y`@ZfO7?hb@qgg{(++=pi^5;xq3n_?QuLYrM@$RV2J_Y=|#D-wCv^;dIQo?srt&p@k(tnpniHBALQ9PedeJ>01dd?q! zfVWmSt09eVV08$shAt7|^fS03?4KadG~aN5doPJ!nuOxQm!Wlu2-NuagJ6j-+C~M# zq@;X~`6a^}R?WuWhYVTgGY=uHPlo*YqsTF;bXmH(2L1?N0-^mEFg57|I85y!rx)MD zTHQiy<@sX4nI`hFKb-bYDP%O&camwRFL6CvZjRvC8ZM{w;8oN%!K-m15Pv8Indei8 z#H=0c+M7AVp1DL{iFDHAi(H`ipcj5tNQS2p1U9~NfZC*$MC#s8zW}31+(KMw+?Nj~vq;LVcAq zG+Qo?yJy;=$@XekmX^WTf9a<^4aeZ~fEb>xk>I&^ZUh?}Ke~E6utgzaFfzOoPT09Z zWaJf?G!_9nz8g{-*%V}NXA$2yuB7#BJ}9XV7=E1V3s)TG!vh;%Mqe<-IIa+bE;CnD zUNQnk#Vr09zYstAdxNw(%k96@5YjH9QP*{9Y!$~u{$0+xRHtFAcQKvHu^-=Gen9<# z#Gt#40Kv5yn8MK9d^5$HrW|2TLWk~7T-ja6ieMOghDRS@Z|8uoEzZuN(Zw=UqEQyaYLE6vhdz@6SjS@!d3d4P}IGLYTf=p z_W#>U-A*rtmjwb4(UZmYb|o0$JbU%uPr<>Ce%crr1|uueaEntXrN_SuKJ1%`4ml;Zs=4t^xbAC(eoa*UUzj6#n zx<(h3ZHLeSO&V6mxhsBO;%=i4h=}kK*zHsci_%0;Si~4QN8Sm7Pql+(Pad85U<;^@ z+2dk}M$eQzXt`V-x5%C2_@^fD%yk)xPd>&7tHmO_U_GvozW@aryy2};H)zznB^4*s zF^!*$#@^Sk>4z$gtkQv{m$cDfX%0*eG6QFCjxnZW4hNGI@OymUhINdao&UkGg1TqeN{*O)?U7Vq-|LHpHa!Q)TOsd%&9^eUg8Dwcy4cH`rbl z3ML(UV5Pbh+{`=3^a+21hzBaX22*En^vuNjrVS*)`993L5&-b07+ME=!E)U-k~@$_ zceQ%Jy|+Vjr^HKvN`D1ELr92>t@?`(bUH!R_ZMwFG|Kge6M=GHze02g-G9vs!onVq zj_I|yN6P~{GQvnW$4z_DX9cRE|B-1v5l|#{U%+t+(Wo#H6=T$}rMZTFwZ4dzxzlm_ z%^>(#o(zp1i*Vpb4>KX&2u?J;r!s+@v-^DkdLN%H(1{DCJu%)i)-xFuzqJwvOK((2 zYi0`Lj-Y~OJSN{PA=&r86Qf7QRP4X|V6B=CH{NTK4b`Pkc)yR zbQsutBh@6lS{-i_M+;d zQYee{x+UQliG6;7G%76-wuJYR6q!hJeeDVQ>(q3z>TWu0(zBuqe8VB@qd5L5Qvm+j z>#)tJnZ{^JK}`F6)H}-gHIkCRa`Ysfr#2G{UU7V-TuZ^g;{c5RA&)BCO(1CQc(9aD z5EvHu)2zwcNwJQ@o|`829OhH&yS_wMs37rcs= z2xf%kA@A22m@q|{1isvZ3m)A7xllRA`;`+cF$sg6Z=--Y_YeC$l~Hn*3Tw1v4(wLo zCaz!c*v-rYIpYiDc40IP81aR@n)m5#p^4;vgA>TaF)-tGJFTk8qv7j1$c|8iTWlya z>^Vb@w%fs)(PXGrHivfyZeh-X3^2HqNt!$^qs6n^pgb!Uq^C$>^ygvJ$UDQIZjb^@ zTMx`~*+su^d_k<;qCsxh3w3ljcKRYw6kdJ{H4o@vq3UgXbDhh41el|LHldjtvLIy{ z1G@~%sGQVP@<9C-U2Z!{R&BdWy?=OM_4V6OCz6Z8!JXhxMvZx=#L5l{f065o-Q%GH5G+TjcKRVd%F487+U4- zM)jl45O=f=rbP}>UzY)T=kk8A{;B}W>{M~TAc0n#(}KvB&%|>62I$Vp09#KJcDi{r z)x0wkhg_|&>XI?k6>fp&haThB$&sYDSQeg~_D0TlRw!89*-rhuBC%;l3SHVP0mjoR>BF#gI&P8x&wD)ter-53Car*Z zt~MYPokq2*xIXc%eOM>;R*==2O}5?qLI#Vip}Rl-=F) zegOS^NC^z*_7T||QKVS@JT(p$f#(lz5{ZC6G{or&6p5Y^oSVjVW7Vwj;JiKLqkJeV zQtN@6)+eFAGllNj)lP2SIuGZ#uImXfg({6JpeRF{>`2w*DW)Fbtp7=5iT` zq8xK@G5jp(W&Tq3yETU1Eytyoo9RKtccg{npvmUzIL>1$dU$VxWB-+c)|YUUv@wM# z7a!yI;!CtsCJAS#a!hI|z@EbcIPLTj+TMMDbV@YR!WW+K&t0Cr8*C&Z)~%?o7{nLe z5l7$LU4^G^Wm5lF_sI62)1XgCg!#38H$)e87b zVO%G&O&rIzlnEs7|08anP5-CpJp8Ht{y1)DMkzu`(vl>hxaWPdQ(B@$1Eo~bLPd&{ zy|+-Z5-BU|p7%vo$S5O9StUwKN~OMjpWk1=ocH_ndOo4mWhtmSJSTtVi9kqG z2t9VBS8u$25$b{~z0Iz?OdJy@*ykUZ~xh2oqs zS{ghPK1RLdgl#%YyR$yhSLQ$9=i^>_&*&jFDKCU2TmBMZjV$yytjL}I)`qil6Y+Zc zb6RU~5l!x7C=bAx=i4_QP`*vfW3LAf4 zErIVhLO|s3ca9rPXMGxdh4y??ba}NQNUl+UH7hk>qG&x!*sB8tk989JXhH7p{6cgQ zmxIj3`B0Np3a*=O<3)FUFT`V&YH-jfi>CFd09&w#+Wb{z?W6O!!tt}Q==K>9d}Ijo zmaXDS>K72JJ+s(D*8I5m*cdSk_(6iU$-&KSlNfesGw2$Ykm62f?5|aa3Eu{I;h4v` zFb9ZefjuYnPX*n%HkOb;0k)xmAXivc3co0bvg<4UkhT0XxyqmRfaDfQ)H?T!rIYm> z6dw)3=o)h(k*r4F9%a+`7REc=ItN-P57)V?1G*x8QTVqD3O(*)IRwXn!LCR+`u4D{ zNlOQwc)>!BcoZlc`n6tiZ(4b7Y6~tnFq5nEIu<-HW`L|`B1rm6kS>i+EXznwcrV9z zwio)M@}{43bzwFnnEBz-%TF*)y$eH^ivXSEWy|lW;PiHd;bZL-a=^lYwbK48C*5YI zp4n~LEG?49J0I)%O~x4<+Yhz{&L2Tv>VVcx9uc>VGND$DBvL1L}s zV(t?xRw$-3r_xy40^`VY`#`vJ+!oKpzoIfn(_!Cq7k#y*9%t4H!i??rQEkI1`twN| zrfdtLInCGMs2npJESpIe3pc?+-ww*7wF)m>X(b{PY)IsLj;~eH$Oelvh+DwJ9p#;e zmn>y)X2K%)PoN#ln{L1Xl}*H*kJ9z`AHi}LO4S@su_C>SVd{t>&|4hF^&A1WzD8k= z-Ey+cLK>>atB7~WCJYWwg{VMJ`sKhGJhIvfqZLBXJ?k$0DaEgA`yf^K)w*o(6#hkh zZT;YD);kFMZvpmf42^ymX!=6Uk5rW zJm}_*x#+0TMXYz<#M-?fbWeaW<&PEQq^3mRT=~27+H65K=cX8my?qf+zV`*!$p&Kj z{3o^IN@30mDQuU^g;R%Cz%0!{)Nk>mtixs~Q#47AWD7x#;1;SbyqC;vvxenfnIxe4 z5m--hbSFjf^yj z`6&bo@2Gy$vn9LjMtqb1=F6W}DKEwX% zUiKVgoo7MjGi^xn@d8%Z75vPT4|1;gn3??*{VPfTSx9~qZAU4s z8hB+UjU$iNVE)nDXlwPEo^riLJUCnFa%Tzd!EqK0Tg)H7uY3fs0G}B#OGx4_Xm)%xCBKFf3fTFT!>z@9?nN> zqqc?JAbaK~9tq^ZX`66r`n!)f6)y(qgkLbPLV+r;-;N7iF3{rq7Z~$q4SbUxhQyUU ztjLed?7^CYbG6veGj#_V(qy>K+{c(U!NYz>oXK#e2CKb~z{2vsbRuRpoN0CgqrNww z|9t}f8O4H)Q3ExQ=V9|*5#drVKCXtt7;7@AzpP^3d+hJ-1!2b&>UwS*B%E%+smubr zhe1sLZ5G=_dpTYj*^NOml|)DXKRQ-*n5YX&aTm<}NR8}Y(rmYM?41{iNe0WvKd)$V z8yY!k<+3FBz8Z`*y~O~rT%uU{nzUp}0)vQvMF%h9+%a1WnYzlV&CY@=Ra~54--N}w z;}Ec_0p?`Mfrp6zNWExg@^sxG`aJ>c#x+rBUk~eN-)iJ^&tSjZOXNsxR`}!b+2OT*)Y8LwT#(`E8<)8XsYA7A0BD%paVNru~TU( z9&8ukN|;QMkZnQ|l_d ztgs4A_l4p`ng*p!0+{og;R*jfPxt;ECJ)U?j_>Epk?{ zwSIywe?LveG)`fplpKbyQi9l@KT+IWg~}H1CeB@R;A8Yz@?ZR8m~47O`^?x?6>s$V-V-JLEOu%#IrS#Xv zI7~UKfRbl}@rafQ*e>4%rH4*~KqSM5p5;jNTtnehS_YPO1!DT_ov_zfiP#p;CQ=iT z*nh#F%rkWXd;d6yZ#hJ=PMgx77QV0}Zz;SS3&OAFk69z~nJ_Oo1-!0Zpp|zTiRS(| z+`*j*fy{hG*=rFLSH^&^^-)yf|4M$lsdN(fVXi zny#REbD|(+a}ei4_yXozAQ5VwsY5)^BjPAv2TSwUGLCjne0nH^W@(?Nrsk5cJlme6 zymupOwfs;`cpr=~Gx+f729EyaC*<=uF7x8H#yX6M0ib|S2I7J`Mx_)xFv2Hki+1a$=naZBnZ$~mU!BCUgm#yG^z zES~U0?Z@@Y10np>9ZXsCgX}Q3#Bz&gFnIuJiiZO_mmJ0?YEqd0XcydcK1?gR_^?eg z4f=c{QN1$&{^KwqD$%{L`-%xZOyWW+PLq`EYj{s8nH0IiaAHnRvBW=ZL|OJb>f`+o ztwbHcM7NX`RVM@Ye0=fXJ1KlTb1(h&&j>AdH^Sa0mpL;s%mHKBFm>1)c72b5xZB&I zRo4bvYMH(Fe0K=_F9|fSG*U;U8r*MFO;SY|m->Y}bn(p>)Fv_)7u8(fv(wSf4l#V|5D|rUY|`?MS0$s%Bu6SPEj^ z*(`R)A{YITycYnh$>Q)8@ocN4X}Ly>o9 zB;M^Ufw7&A;B3AdrbzP0#oUjkTnI&w0i%Q#wrqRN;M68R%zjWok5yGsC&Rxq zl33tm+zT>6OrXwGA3S3gqC;;IZr*y4?3D6C@1ZLAqFe&bjkbh6*o|@CW_av*IMz0u z!#u@IxF{Awm6=&q*9e#X7yX;KXlz0LEH!+5Y9(4+r8Mb;A_ywHqSXb7An)J}LxL8p z7n{BDi3-DhGZZAN_WHoJmGa<~YX;&5YvIF&E7Y{`FX7o>MC%o{u(HhEU{@kCF4EUz zZc`#&O4kJSC?*s2-I;U_3!z*(^sY&W#x2{j9NI(mz!afe|js26h0_EEh~ zVP;;eL!8(V@cy+juKHd>&i6ke3taAjeAZWFm)*y~FKQ?)w;#@rP2icf`*_JFgY2yq zqLmK|I2v+l@J(SUT#9I*BJ1jLd$1s6nl8r-JyCYj)L-amHpRJexAeCCh^Fwz5?87# zVA-?T_%R?I{#^P%C8o9z5*~(|ujsL6PgaqG*L+apgc3U{N|?QAu{=pUufbi$*HUg* z+l;>V=EIAXw^(dGU)qy=h&oMK!?7}P;$rQC(%I%jgtLnAc_cIa<`FU$e3}Zx`M@~M zAgYWf@}tT$RBzjdFVDJgMCIKHcXSqeuJu*;RaJ`y1`lYswhx@$--%5&4fNlPIot}F z65_994d#;z%gcXvz_^k(4`GXsheBsf&Vu=I@N=%#L3J2_^0zniE3X0($ALy(4<%qfhAV3{`S5m5A=KR$X4{I_kVOk-a2pOR2m9E6WGFbDb;IHWJ*t^S1l;-9S?l~D z)riH_bIgW`_&mHiM-{9tw-UD#!59j%1Y=Cd6EU=RJk6Xc@&r zGIR0or^}?EI3HXlMVZ{`8fY6&0{csOGzZt>lSQp8jerVjx5t6IXigXA!A-_>`;jFW z%5-a(J4_}7GW{M)Dl+zgoQ!NC`771n^qWfPk{Q7h3~S%#UOPMvOok~d9vodOgB`A! zcz81px|FR1WnWcbjr$Rqfd=wJl%IQBA`2@G6KP*_7|C9p4Bn0gRQ{$FrWpib|Je!5 z|6)&%yD@Lzt4r8d?F49uQUi???4g~m|2RqaPlK4EFKOsWLdT0{T#u0|VzvGOWY@od z+g>whWZ4pUw#o`M=H!zQ{ZCNGx0YR#myfy=w`f@Z4PU}%RhTyzZ~ z50t~vZ(0SH4KshMWuvV0W(~dYNIB+QYlYsf({QYP9z1w>629e%qvOl>#8BokU9tKE z{ym`$3-go6i`AyA?x+cdx5I|%S|^k~Yz$?r+f0sp2X5q;Gma%wNc7?39$%h;Cm*HZ zpRWI4DvJ5MR|~Q(Ip$+o-CEkF7(;fo2h-pBXGu%NalL}ABKRO>Cw<3pVYt_ma3pF4 zT872asF-Lhj#AMJ#1$a>-%)(@>@oSy?1c2b33LDS-{7P#@@D-EPA=CUSqJOTS8vPn zN>br-0$$trqnH0_QoM9GHvIa|bm_8jvZ}HCrrch9Je)?=9hS3hTBl*rb`8iE_<~Qw zg=z1SdZ;KbfQQk|^s=ioPWa4(>*8_v?8z7@Keiio3HW1yp%W^_hQe_2UYy|+fH@Ae z<>hl2X1eDDO+Nk!wZ1<@(YQmfsiYQH>&HNQpe2}OD{zPETj~0dboi|r!(@;|z$&5; zR@4u2qGH0a&F6I4aFrj!?LLU2@4`7{hmO*t#*KtGHGy#W&ComN8mamgj4OqgL59y6 zj8Fc^d0n1`d2{8U<#HzZGea4E49;SAUV4l%s=^%aU&YTII%*AXv-`=4+Kps+P&y}a#2BWcZ{nR7(o6tD7DU71 zz(4alYBz3y3Wp(*tF)V8!$iUx=PfwG6Gso-uA&{wt-;tJjPQK%#Lc=rnDrqIHc36e z(Ivv%ZYH0$E9D*-9d3qBR}s3j`wwXpl7cj2UhesCtKikc1~3!~Lhs6tdUKj1Fx5l? zxh~?|+D2X!)Z$~_TJ5mL{1%Sd#Nj|&9oX&5z%@xqu<-98I4X0P+qd;F6jg7gr=8oV zmO&tFyOG84cfJsF!z=ha$_TUggt+1{cDOd_0o<6_03#pbp?%;v>tbgFRN4emSw3^J zw?BZDBRC6^s~bS?)B~(7o`;fEmegA2GYK;gBXw$sop~FuGJ7R{o%{^oHJ5wdG!CZT zBkdWLq0hb6X`D&)xj8Dqjyuj{hdJ_@2<|$VB*bE}xve`vWBG{UOrQ z4^Df`<#4wwWF7iB%CTsvfy2&wV1rmpxzO%jqI&ZR{nh*xKjrU5RZkLvr7m)7q1Fv06hBn+N+jr)pzrAZJdWm#hW+?4)KIpT_4GYfJj)<{FAV@ z6@Xa*o5_)skS59{yF>MG^{w3UnAsnxSW7l~MrVNLS_$?Q&J6Ar-(B=*YY@>ENrC(x zhS6+(9y5Ywp}gEC_)ly;H|ZVo`y?@pxMwBQZ!(MQ*>@2(?vy~#5CQ|i!_+<74!1Ki z%wM#NI=(#(E8Yab>7XDoL!f}g^F9&=_j%)`fNoS|jpN2zZFuzgJjN&8W!K zDbJ-x$X)aZWK<1NXR;12xwAlH=pJbFFgYiKVe*cTLp>6L;rEPA3_a1oYWo-oSq~&3 z5&N)ZeLE534}`P2skGp|DCzxK0Do0iL54dUA2MuaZ{cAQrSJfiMs5T8*KsuZShz-= zvP9c94*(F&kfTAPf*86AlQ4yYv-xnioZ8=C^Z< z)tiW^$p>;}r8qb-&XaHN+erD*eCX^DXSc|QQx%6&$X1AkIsY=@dGsgL?-c}%mjil^ zvS0BgdlT1qT|C`47DFwM|7F?wHjz^uT!_@ai;D|JNn4{IoLwyoGd+&dEswS6QVVB# z*>@ocsTc?Tk@YZjLj3<-Vv2p*tVPT_j_RGmKk0ioOV{0l-@C5j@vl;d=@0SAM;{d4 zeh=NfpRr_Bc&O^&FfpE<303D`!j`ru$m^Uz&u$E)0kh-LX_+cj<2c}AGiNM0Vpuk< zEri=5bwJ9xiwx|!4NudRAiM1)Hs;)7xooyVmw)S^H=-VeN_$C(;!)V`cLWtw57X+T zUJ&S~oabkL!cxH;T6o_YR4u&VODU4A-Vvx?*G_fyo5_E3He$mn3*bl}L&bs+mQ?>t z9NZB_qAMlY_l@fySA-YethNW2=(G6wNESSF%^(u#vp`xv3U@Yg@K}UDex0L6E{1;u z!}}3vjXPO0^&2^Bd9=aRrxZ8uuA+|77l7~iA7EWj#pP#SgTsAhW_aKTe8_)*?*-~e z$woeWut11w*DS>C@=hUXLJAPwGD$tx1)%=Lct|Lkfm@j!xc2itSnar-eoS48BlfOf ze)Srt3{<1&og|j%qBU^u+-npvXd$*H>zI2_H6)K}!_kyHlG!B6evz~Rd|SSgm@SuZ z*z6Lja@DyPtPx^1eTJpHvtZS9C3dVGB&X~O!R}KBobKwN%Qo#LT6HV2W≈tp1JO zLjI&{ohe=8JOd2RnvkbIR}$WqekdCBM7dva1lD?hc(6S=&1Bi0{xT!G6x7&B|FpS( zA{DWMB$MfzsW9B50j6^AaQjgS__{M1-Zwo0<3H<&i?1n`{*1y~Z}*Yd>k+KR6Mpz| z9~*$#r{4H+72hYf5%uMt$#L#O_{H&{=SAgVzlI{rSoe-dhr6Lc&nT{Eg`swI6vjyU zqQDiV)1m1~u`Qa&zP6%ES{N=G;|jac?uBQ*R)OS(R^q;CoN9#5hKX5+;74WyF?#-( zTAZg4F+%{gFBqb6KrX#dxCP}6rm6WCV+7fBfl@0#3{wXFOOklw2S10yMT7+ zBQmz?FXx)oUvxS9g?R@feK;h|CGS`y`DigM^Kt=)0X>Fg&qKbuu)%*o1pX%704M98 z7+4aFJ0hRb>7|b3=5;fYeIXsq_x6xmeav}e(+(5QjoBCxKq^GMVAH{vAg(dliofY)#8ZF+`{&lS_xgNw`faWx}CgTwMnPZ~jmp$8#Xka0suO#h!|>a0k_F#5MQOmFpw(Xr(YJ@Z`p$D`l8{*7S~-oo-5$6?d8bI@@=g#7k5qKzYhWRc!{`o+2$ zY!^9_nl0y`s&pFUje;?A!!agTyMrX^xIlzw0;r!CWWDA3K#}Mr5M7|eg}&2hR3b$ zVR-PS;CM6~hBLyUB)b78m>GEC4hbS0!@_~O4OH9y3s@;-(6gL);6L`0b4Pp)JFfB? z?#mMZ6aK$=N_jU?lz#-0mAgU6F90GAFpltJS(x!YhaP(!3BJ5>OrLBXct`!Ax$_O^ zm~$I5FP@K5kK$?akMoe1Aj)3rcO81aSCS{5qS(Sxg}*79xW|df;B0JTZ80n-bH2pm za#tUey*!89c3g@ZhZs+)elg?x>|%NeLzF6~Fngqh3`2S|UJ#4~3+@4M{2PdC_AP;? z^L2XDq5^2SQUbp!EJnjo2~c*uN-eVM=(bdE__EcroR0c{ne8dkWcL!>S4Uv{Ov&{D+o<*0@b@iFtCab0|Fis-m!K%vBC%=FXh3Rr2SaM+)Ir&-30ZOTj))Rznpd^ z7kFfK0$OGUV#9|%x@U_ZwF!OLdCZgu1h?vq<~+nlf@OIB&r4w69w%;^nPi2N0Vw)M zLCBOUaNn!K`NLVHn6(^>t$8_9Eas???k8KU`Z#dOi*6hVMA_N`uzkLyEOKauo`ls^ z=qP1AC-=;eP6nf_>nl2XWdRf)(B$OnF+HlW!z66RaU8MyL+?8mlTh{w93AeWgG_(9 zlz%{nM}sdP3C8@>z06tUs-J*wdn55KlEZ>Kej z$S$$v^uN<}SUR7NadAgM-#EkfO`XFYw~xTBuLH5Kng?!*lp{yN8MBKF;YR-lu;sfC zgX19>z?w}>j_pU+yf}twq5wf}o>RV!dm+EUlFGd)XALDsV~G^dp)o7ycXGh^Z$J;3 zx5Aem8DLT@Sbs#JZw#cFCIs+RrxihE_wha^HfpAkp~2CdC_%7 zgLH3G8XnGjOgdboNot-10vp2D@XRCWcMUT;S5c#wv#M`-FbzmB!}RVdVEaT?o(E2!2dYt!1==bI^Po+F#8_ zR$eKInV-b!d%Bni&&WG$2Xog-!vD0ixc4iP(I}UV4Pw?* zD)k{)_Uy(+kDIXE#}Bujcn#|lq|r|Q9y-zAP-j-gYS;FLnmTb-jEy$qTXDpHUrSJ> zA_gr^T?A$-j~*K~!fP*oqIuhtTzC9T#K*!wJLw1Buc7A|3@{+)+>hSlVwkQnY!PXGaxVDh6~ zm7L{yj4S^dkQC2USUndxGkgO{W8rcr6x)QCPxZpCkWFAE$isd+T}A5_32>t$887OP z3Y{e`jU3Kx4!>swywNO#P<1XA%}e1_1dfsmSD1T2%w_zRmQI>wGT={CFcu~VqP(U& zO>K6?uTRdhMqfnIv^{%p*ZNa<-Ln$3^DbgeXD!XWV}om?({c4@DQK?~qO}zQOfyxoHl2smDv&akUaUa=38&Ob~05FA4r+*OHq`L3HLZB^0kdjoK=z zsQBeHsCQ>j?@k}Gq$8C^u2-ODF&0RhBth>AGZ$7I3xny@hc3O%;J}W1RaserT{{8Q5GfgRH6$u(gZB`JY8#JmEe}Dc-@Ao(hOi<~Q8;TbzzP7$J6B7ITfh zyd?9!H^U*xt@t#+TaUk|0RP?Z#lv%AiCNMO!kMK{AoxB+)iaDTp9s#g>LJ_}Ai?E5 zaE)%V^&`jA4nqBy11$45g4pku;9LG48a9F~?y(FUN|FG{f=F=uvXNZYS0gjMR?&+e zieNzK0JS@|0al2u!vQk^cJ8Vt92M(0YaTp@143V@=e{6JP`?IQxm5tqK2vQbZ&7hA76B6i=iO_-*Aw7m1q zZO)U`i@6O``%p`0C&`*vgdrSv{Fo{ZM{LX~Ga^JmhOg{sTY?`Crs3$h#oVvE0%5J> zS(rb(n7eiRP25~x2Qg=Jn0!(eye_$h)~-kJy&}Vqbu|JUUyEE1A-KOzpPp8J!m%#l z=e9PFq5KP3u0vohEf}q!&zKDO%Nh2#(IXAGr9zmmD8L?H)Jql4w6vlQ|ImqV|fIP~qh1pX@Xi1gEXW__+_3~& zlHSACZy{y#BtH|s7JuAVwF`p68>yRMDKsxyf|XmUF;L|fhByw9AfFPDbf1rj8$9W~ zl2e%QAc2frpNj%nzO*oNl;)XxFdnOIligUxWa z64M#VT!pzz$mK!B2kaPA;*R~BiEUD;m>*`y;k3@eHHUp^UfW;B^UH$oF{|O^eKRX-e z^grX+wKm{FBFFly5rLJz?|{Sz)5odVk0;x|&@I+&kTUd${+M+N!dAD?LFa9xiE-ob z(OPPQ_V{#i4Zd9dg!8UP4=of;scE?s&H8JI>SB?s0bPb$p|=-yt``9PBq{EAj0?;O zQHCL_Ef{Q~ifIv*blR(t^o;+ZFH6(0=}jBmdAWy9So#rZ#z7}lXh`33=1&n)l6pA7{VO{ML7%YgugbiD{+0_ixjPSL*@1lJOg@?9bq}lI0*hZGf*! zc4f_h6<^ujT;7HLJN#4rUUEG!bwr3uGz*JnK0Pz4eaMnxG7ut%6{ac`AOKSBa1Q zBk=0EEPL?eGI$cd1|uKx(~+&}aBJiO)00xel_ENjZl(+8qM17B>qJzJFo3l$4uGmn zHDui@g3+}RFpE7vJtL0OT_0j#oqPq0$9yfm(Obs7aOpd?PNvZhr%UxTxvCi6$-})m zB>}&USFv3^+^AAV3wB>TPOg`Ev!oWz9&zcLoq7TsL z{S}fBlZjOP1k^8%#d{^0MB-y6Bo^JmPcn1ht;7S!Ir4)eDb~T{>gM69jf1S|Y)u&7 zQ~;{0!?CITHfnVA!%=A$Sm`CjZFCpK_2*uYG1q=_UaNwxUS$l<-NDG+`wj+VILx`| zh3htpaxeNYyex(jBieHf_xK(n(euoS*)JtrbgY7$YJLDtU9<4WPjN_mA&%DRqIBCe zEjsX3iF;Y-2nHH3bK3R4Su>BnAoYnkAkXAQt=79U8K!R(H_S$rh#Z{yD8YWfbn>z@ zuVQh*3^>;@PPM*uk(JvE@N}RHR^+VBDg;zNiSB=qGE;=xDsujw80i>uCD64!_(jW2p)yvO<%tgYk^Qa_hzF>@1N(B*mPW z1uQM0IS2W<6^q2M!TAQAGVx&UiDfivz=tEOBLcA| z1sj}J_6Bd40+bgJ0sC}Ma69f*?x7coHFFTlB`U~*rN(5|v>+-mo$+1mv$@&};z`~z zC6KlW!TZ72$Xs$BAHBA~)TM=V=hNS0lM3TpKfV+3>LPSmn1lO0B0wE{;E{Vep8RqL zReWbr=k>j~Ui=a$Jqn_I8Ovb0JB?!IX;^cKK$~e4t2;f4wbAJg46!${CifcQ%2P$=M%h zOMx0+hI8r5!+frkKvzbYanP&<)8JF^;L#W857K}p{_n`%H(0(-?k=cnAD{-ZYv3s} zBdDK0jso4~bonI1b6+C{LN~9&^zCwJo42W5ius`{ZKgnAu(Mbgvya!WYf!kjyEX5q{4NY5U`%U zkW+;1zhrUVfiw~yunxu^E`?d{fspswM*w>xKG{E=q#Tzm#^+NYw_+5b2@j(sOdi*3L&qM1y4DWgj$ z58C1nDt=EMhmKg&P9{^)8+4bd6<=T+(Rr}Xdzfr6$R$>6*7M z>A%53hG&)z)zg~jucHnloEUI8z5$9eH6e3^aj?9zMK5I@&RmsKu&^|aHST;1ztm(f z{xF8r@{;khHq8c;+u2mTVlKL`JBz7v5uT)n;4kk)u-{T@t~_zC96(P z)k7L~ytqQ#UxbmA#S0;Rbr?;nzXdMo$tcxcNnX{xCZ#f?WMEecDuqRniMmrLe%KGy zpVu+G`3THgzaOOHHv=$lEuQzfWKp{&x#Hx6`_o5B*Tq6C;t_)rJs~tGT9~%38={Tl zW1KhoZ>auUXH>p&kDf+`D>k%-43F)`_{>O{CF+eMWG!UYse|Sd8Q9BVJ{xmwacgA* ziu>Dx<>+&|bIn!Yxs#7_d`7e={T9?3n!wV1JMn2U(@)=8&3a?%1z%qmvm673!RtpM z{j)V5)@gmC^*$T%=Ys$cYchdLhvOmk$OC|z!r*Rm7$4HhaOPP!eUWe-L$wF#h0Zu) zlV(77ytqv@X6*)vZa>hp8=!VO9pK(!MHF_9h4My0*!iyj*uR)}^QjQ{>Fk4H%lV+x zWq@ur69e9ND{WG`9H&Dfpcfy4+n)N7n!;Cl+sPN=@NqXeSk?rG`%Yrc zB5{0c#pKA=%^-h|iG%gcEhKgWaCpp2SsDj`nyMgV!Yd~yh zE*|;HrH3Dka^{VcU~4}w+-dcvhECcb7_38$OD9N+Lm-SUH>EUuKM8#j2bV(jqWOj8 zP-n@7n{rtscfLKW+IRtVB$%D!S}QQAQbIZ3>*TMcER1`|!`@rVDLN!l;s1`~@iJ)= zS#uO5eP7ZqqU(u@Z#aE60waV8=yAviHtH`lvsLxW`yxRvsaD zHmkvW7dxoEI}591TgaU9He&ftQ##SN9yFG|>DHxNa>*wLUfuP># zWG^XrvJ&s9o~Gl+@4)j*Yw&Dc5S&nEbH>+9k>{(g;qw6-F!?5lO`rKuDZm|F4GhVI zi#l;hT+Vb(O@TjW4sNWwKpOPxux9==`EUL!$PJESoc6X@WFi4MOPf(>_Z`A<;7}-y zV6rh{;CFWpk+^+KZ=Zob<~CoYuKfY<^G-Z5^l(KtBV!skA%S9##MyDX0!fp{C2)Jw zN)vLWIc8a#%lAHMhlcze$ZFCATkn%4ib9Gz4hYhiRb+Acx@PI*kgvmejd0$d;`N-sAM(dpT?FTFS>2{EqrG=Opa&t z(o^qZ@w7uQoa^>4@6l@l^Vqj|OO&7RdK-|c;7XXB;{jRwru3%k8`0m-lRoX92~q3? zxN7Swc;BZ1ei6p#GIbClM(;yr;4|>~C=2{M(n!>mFjm@v>%?G{B*bs+WhGT*Lfxfe zCIfT~O+MM81!o79baBIfrHz~krMp;U@RoWF+Cg0l2c={Q=nZQ_cTWJehp&RZZw&us zMHuEE7vXXrY$EdQ0q}kOCxrV4NzKqpDx;K1C-;n#g>LcipS%Ipe0i5_5ai*GFgdW7 zcc)=2xCU0>36z#$L-vQ6Tw8l*tUW>S@KS(;VNd!>W)pdN!xtiBlJLbfKd9r?20f#8 z*ndEot8ze=TQg%5>)UuT=F)l^Fi#R!)vv^>QDL0VUMFES?Pj^B zhnYfUOrpDtmh;w-A>?DW-e04 z*0EGPEB+W2QXN=N?%dF`cH0Qt##;K}bs&jwv%srizItxsr_ue!D$*<*f%{(tz^aFj z@#4NFd=VBvdQ2CC>i6wr#fvaVmQ}(#69sTnL4ffVg%Iu0QDWDA7Ui|gairre8vT)m zdppLVx6ukFq$B9XrzNQP*$+CsY-y&TB)8Q#7XQIZIC4UU$u-*J-asR8e53+D_bGx? z<1fwu%XF$TxPo?v@v{q$RM9sUS>XAbpU5^4bctkqleyuvQ0WIK&QSm?YQPzS>2%0R z2TWviV0(rDaXG>T)wz^4xo`@$Y>Nlu=%w6<*AKzrN;Wf_WoGijW#!rD?Wk~ZA$4Q^ zFIyG%p^Q@&NPV~q!*mN|{jp~Dg#6rbF-Ls)@;>K7=?3PFp$i+c*P&qnh4^WG>X{i# zZVLv(fe}rxlzGqcdHI(!tP&3ce&Kj;mmfXY|A7SbB*2d-cWC@}7RGgW*x%#r;P9!} z7`r8w=pFpas%4ziSrL-lTw*s-QfE~UT)cla z_jc)NJaBC`4E^ltvNWi@^L4AXmF;3;z)T}2#pPYKo|@>N-{IW=Et36 zGcz-&3pfNtO=7?i@WD?;I#8AUj*jOxgG7EV?2P$Ddf#oOVK1jhg~>b69O7aBGYrA4 zpv;}{ya572S@iyN50;4d!@%}9*nVz1ta}$oHP>!M*&E+L&SozBP~1nqrbt8M+fF=R z_nzS04|KNI12S`NJxu4@;*6m%C|+fVgNti8c5(B#=UE}x)|m#~B0uSc>hM+UnhK_v;GK71n}WWnXan9*DQ|TZq-d z%W%Qw3)Vzf!WA0}Sh)BhnyQ8(U!fZ`M)i^ZJfw+9^D*eOVOaaJejFa#VkkyMn)sg@ zEI(94^MiQdURD5p_iF}txHNCH4SEDd_wXzD1`Cmg*eoJzKVYV$} z^31#JS#UDwBgZ#E5sPwy(d7)oYT}&$gHs6*rMn3X9i%brF$aGtFy7OHb`XC#6in<+ zqU-M)tT#F)a32eBIt&oOW6H4(WtLVG0VqC-&&ee6Gt zrUOx|v?mGZeL;`4R?dQi_X)5)--OaKGku(5IO%2edARtOATDHj(#223>A1)qm|N(= zy6WU#J}^|u3O*@>qe2O!^1VN`pY@5H`BP7FUJud@bCmJsxp!3amLluO4q?J0F2+4` zdp36VS^&uxLFbTesyF8ueEF@3vpH@!Uf_$%U-7W-4S3NZQ4{iy&ztcP+7P*uQ%t^M z4*Btg=>)m`1wOaF^83D9$Xr_%!wud8qg#IvcoYkL3CuaQT?{v@k0cLbed&DbBBr~) z2fLi+al^(Ia&>YCv51KdJzH=b-1DmbN70$a)%bN`xIrVOlA%Z{LJVB``CY2u@Dj(KoV^~0?QpQU?Qceyv+`PUmOygW(HUuoXR{_i+{V>i=Sr^fe=X!V;zqxHc8WJAHOJKNj2Fl&e zfh8wyz^bE`q}8p2xTsyHQ_uXss-|#UHh!5AOuSAw+oM76dm>28DWDa9M@aMf7pw~? zmw3Iuf_3Lqpq0-UkFC%`AKz%4UpSu#9jJtbqR!+`To3Cf(S$t{6X5i33q0}frLTl< zQOo_w(AL2+C|lV%tKJ$Cv#Xn#zUL5xMVMmMa$7X}slhq`O`t5@0DaC4K~qgO5&57_ zyxs=GGDkvw)U@Epth2kDTaayLT3id5UIPLy3SSejW1cGAeGw&hvi#LH0 z!vUf#8i0O|B9N2p139iDVAteJ`R`DoXYT-KhYe}`U?Fs}K5EC8w@I|-RcH`!QTFUQu1>?&R?yL1?PInuXA#!cJ&caSnY&~QMPD$ zU?GiO?Tu4`4@_jwV#87qy60djb*$s(p4RDt56y^E*1BVRQZy8umgin|@TJ826ZH@{ zgj{xB)nd%U2XDoZqbmTC&P{ZCmnlu#9}eTHu?CF?jX-PqM1SE0m;>gXx^_2H!VkT#uwHtBK;oK^&Ehw z9Z{8~8iIz=4IDKiVbBtN%Gveo2%Qq*&E7ToNWha4A|A2`talto_3)>7Wg-C=ySu>L zVSfmZ^gtoGja2w_5b>{6MUg*jzPM!`dRr+2@Jz5`Um2w9-GRATKX8QI^$gtv7`wpc z>azENy~Y*7iE5=jqmRfpn>d`M)5yz7qn@R0umH1?O% z6Q}aw=Y_vibd2>#JQn6AuI1;1Ub)Bk-u1>j7jaNBdqw^0R4_B(ER-0Ea<%W9;e5YS zs5svjvf|S?_x?N0`TehhtQx$^Bsy2)`Q!wMWu1wyc0R(wUMJ9AD8|uutB29dW!OCM z7#C)U;*x_Jr2NBfc(gkJ3{z)f@;x08zUNB!YhEEfM^;1FEtWUF?=0lAGl|o?MnUOs zFLAu8ih>Kz0hX~H`i(=dRL%+BKB{13>JQ`L6R&ZsfT4=R8Su6$1e)^)$mU8}d{Q_A z)vsx>J=kjEU>ygBpDJO-+{^G~D1n$fxel&16J-BCe(sL+dh)ZX2lhLQkw^d4p`NoD zJs{}K_L{nhSLHEXeTEYFt{P|dwOOO;hDexJstbA#TdN zLhs=)ocSjg)0)m>Y}y65_bV3ORIypazeV)M-{sJ9;u_%>_s3&LQ&8g5JdBDkre}pm zz~rDf+@1a(I%KzTnhZaK*s0INlAF)u7%Aa^g9|=dpU(DCCrQblcw!?Q3BsrE5dOzm zu(2@+Q|`0R$VDQYV}9Ssj>ef0rxi8PY0e-@CP1^irLLjw%z&8F|)gyQ3a6)gLF2G@PS0^a6sMxB8(pq2gt z59kiU(p7qpACwCRSFoAyFJ-8&^9H6T`{KE?d*Gj(7DRu2LH5jhN-xh!An}$9(T!UG z&eFS4_USUpSrp0e{f%ck8EPP(u1}8b&x7;Q#bA+LK?gS-rJ1_~xPObEQxz>0?u?qd zxVP>)b{&$Y6Q9pO+RGA>=W+`*I^$rQ-gZufWImNF?Plj{ag6J?m$Wx^4#@AYguINY zXtAP=?X%25mm`;PqHrhkSk4ZG>n}sY!g_f1sTqofSr$rg9(3ey#Wz2su;VNjtnQzn z$JsqueTfcNx^kFh=qZ!@RSY&L`H{PSOzF}K?NsoLB)GxwSnqpUaC0*+O2)FCA^U6SW2cIkhZDuu;ad}(4e_ub`{%ixi-|lnj-F%>5 zycIko6u4&ki(p2dGZMXEj@r={Y_5{z_GZ_U<5O}`E$jsqmh{Ct#z9QtNyPo~4q@)CFcG@Kb2l7l^#Z3;7DQFb5GMI_z-!(G zDE3JtANt>sjC-kI<~xsdXdWb@gL@2OExD*@$Fc!_mlL~f_h61~3$3fOLf{P)b|sy2@u3*^YI+$t`8o<(GR<-HlpgO?x)prqo1k?uESr69H2rkGjVSMa zPekR+(PiiXJc+g2#tBtjy?k3(yjI-?{%Om1oAfseQ^ zZui>(pWN=!iLs5CaDp-gu?!r2z6M;6$6)`{VwAeDkj(Sx1OfRQAheT5wgpGx#IZ5D z3=ZG|&5Ml23st(lC;=+`HUjAmCmXz0;>Wk+^ym*YShXk?4#xXI)2}^r$aSqSS4qE~=%xeq*6_457h)_gkpAh#_}=~%F`vH?um14igzs&Ik_DkSZ~F*| zu2@1#lLX;$2fHivNr7pXuE9F-<3G+DbevUB22VVpqpJS!g3at7 zSTasmwp5ZI5jtpF%)V3qYGP2=NoY8!NNjg5r$K9t!O-vtdYZ{Y$J>ppqu~%vyO>GK zB2qYGN$0VlnU4v|aD}XIN3eVo%Zkmd0nL``iQ#A43ONX&7E`ws0x`2u3FESBC7!SnplMju~D!=x|nVg z(;-?rByc~^6(o2HxNisRe4gazE@U}gukEUtNX8cfbxz{O1?>ju^)g)h8h&~P-AQZe zb22gg2548BVsDKvHm|fL19ihN_Tn@6oW4m*s|j$sy*V}C9uSqH?d16UP$+TqMo4NU zQ&;k1Yq|@B$<%>ostH7IIuE4AgWjr&1;GXl3>wjdzSURh=HgRur>hb=7VG2K$QlTV zna%PZ1>oTEGze4n!fL7rMjPEhD8wGJHs-;roDFoB_gqwG`^bFeG8oQ(2E4qYkt^ws z-Pv8$g?c9B;`<=z7Pt*s^NU#DZvb5#!4H|Dvta0@H{@Kg!kZ_g;k-c$XgQanvK|+j zCTGD@SqT(9k&7=S=W~ST|1wzWHizro2iW23f$Cqj#j4evC$wTSzf<0S^Ftc#jA&dDt76=N_O0YF>lE!^C$0=<)f#+0? z3+)Q4LObKpO{fD8?|VU3ir!%?x8J2<%U_U7{?ed1%Lay3*!@zXJ#UFxKiffEMhx0> zNcM|Na?(hbxoY$s(#FJi25ChwvEu@rm-`>88m7Ts>p`mh!XHA^ml9&H3s#rTKuwS( zSu{%o1UH`mJk5G>d{<%0EN8l5?Gk7Yf53DN)Ka5fJLn9FM)4QMMEuzzqIcJi9$jpX zQBo1G`QaK=Tw4qM<-<&UnGZS8ZH(EjY&UtQ2`H^krm+#7bm`iKbXQCz$lr9KL*EqP z(wk_s9oImWujLpbcZi;Ry%)3=7SMs-e)8A*VH*0L6lPr+sW{uOx|*}y9M|9RBW-26T+_va7+MZunovFN zG>nDF&{4uwna#^e*~2K3A;y}0+X3c{cr*|E|Ie6vNeHJYrtX^0V^bU3MTiiLmc-b}vw7(#5pNuA-tn11MN7z(}#b z*M|k=RCY299xT0DJt@@z_liAn(G?v|>S6K0q3azKh)uic~3y#Lkp9bQcNRuOHjVjK|0q`guD8_6Fi-;<6O&~M}LNez(AcF zW>+jg-%%Odx_1*3@K1?Oa*}XGg$(Z3@L^&HKQQ~kBJhg;0hm1!jQKKI^y9Y>lAvKv zN2VD-pt34&R@h@)8vT=`y+03coTSL)8&9k;--vSr#JJmsZh_{TbNJOZ5O}TkNy1q* zUda1O+;BUW+1D;X&MY8!>Z%c#N~gi}^Zc;5md z-uw6Byq{7w^n3Rs22Y>?x3#Kd$vPWK~`i}!H|^- zS*>{wm58~U!wQBEwKCz^ddzCJAN6L!m*enEooteFh1jsq#BozG zyx;0dq(qK`;lFvXRO&sM*0q6n+sDAGsb37docQUttA03EahLo(^@0&;Dj);t%fV;z z9_M=GdZyF(14buialBr)(E*z#daHX5<(PfI&N@D9bKC_B?kU4D;sGN z3%z#pGkKnv3pWR^qkDS?Zn;{>$QZj}57!Ap2L-`Uw+c^cJ5#=dND@BG2P=(*4BAtk zfohNmNE<&kkQn%eFTbYKc0+Y^=`KRoXZf6DvvN+!c{X!2?;+`)6N!JGv7x5D%Bb=# z82AnC;hLu{o{{oJ``^jXezJr5EUN}h#1I`(zpw8=lDKWG`O zDrBEU)_-8z{42owV1-8^4o+_TL!K7KlC$x4@ZfenOmUMy?UG>ZORAtJx;!B4TrDvHdkgB{l1ouz*J2tv zUP3PMfIZVaG+2@E1&fxh0Ns{dyo3dA23`t-M0MFEI30MDsdgVh59GsSx24>B`G4RR zyZg#85QVwwBSbaU3|$ni;i+$zAY4?Mw|y4toQ+X}dbbDg$EXcgZpma4L%A5SI1oer z`r;}9SDNye_1-_r<}^7k!|Wd{gYDov=5uE$7@K=SZS`xMkgY_}T@8i77qIsUa%5Wz z;qurC&fb9-G@ks43q1j~?nmN_mJN6^d?6lReT&LO-UPApx6x)#eD%ju!Nku)p01d^ zjC)X359cmY0M0pvwk z7W~v9nq-tjU3vw$&&njo&7lI!`h6Qeb>|@8{wGk8`GYJxXoL+*elq-rHh=}Y)7=>3 z4O;EZAQ#mNL$kZ9)r)Ua>0Vp7DZUqzOQ!SeEHC1wG+TJ?lm)lh`-T1a0i?6;Vt*4q z_^6JO{WD`YVP3V6lJ}MQP|nA55j}_Pl{&Qeg(Virj{V&?PAO_JDreK3hic4JnU;apHF}2G!gnJDs$7 zpAVl#^VnEGn+(Qb-)8*eAPaYMU&7F$Xqwp0?t31JqUmNfW0kmoonN_-3Zo3-(y;`K zHc7w~Nh4TTTtS4(lR4V4zaeSO0?t}>TTFT^gdem*InS;6kwTyu|pQnJ%-Rt!5;sUTUR>gwaFF1q;$hx}Okl#3q{LC*V+cG|q zNKq|P75bX=e@-Pb&EimZem%Bai^F4gM>(q+_3(^eI8|wU4}Wy-VSCvha{o>dr+WP^ z3>#HOd8G-u&Q6n_eBOlHi!8dfpkGu5tCkdWqbOTN=3W9gOYG`3}vHF~1H&qpiB-Y@| z+^e64yNrskDTK`_*}a9WT36xoy)3vq#|)dTgACHn%K|AZf+cAm(O`52T6-OZ8_hgA zvoW5TqBWg&_SSOz-02Qy!phlwOBbCjc^rhNbP%`WVPwUF8rbZ52%;99z>O86+{^8U zSdL~KJ}}CJIopKwO&9gUx8c{=FSLO%>+nOn2Fg;O z;pwXucx!wL@8x4FP<9T1$yvg@JuK7Huwj5^A3p&1-`B#QbM<&L<}dE@_azy@>13&& z6h_{-jeTtI{)Uqdn&f_mjUv*#(j&nPGd-Jb`p$B));-7MEGb^AT@Fm|i@+>tcj9+Z zj%H6v^PI<5GoN=a2JI(i_%cQZ75V0q94jGeck=c_0V0_vhsty`8~ZV>W*h9|C?sZ?NBe4k&(H&TVVC zO}g0odV2Y0Dm3OvKbrSo%Txn;G&~;z9UY)=_DcH9Fdmxb>hkt|W%p55C*g`?9_N=$ zG_JX)3%d+g5~<`2nlgV7J-@}{!PXwgdHIppMQlYm_y5S?whp*bGl(`~Nw83GJ6$?{ z7~N9%xq6|KZ0E=Z@1&=|m;2IqpfVeLpR;WKJa^oBtc4o2euP2cow!Cc%D`%2Fe)$k z!BjYTlfT`sY09yfYMq|~JT5k{p8ODr>qc)8qYf1iE`EjT`H!Ic=r_H(%KV!+QeAbFW&m;q!aq7`F-^{`UYhl|{HIr?g>h%@Y#8c|Y*o z+z(!M%?#Jf6kipx{_l=xJfbPc-TL!AMX60Zt9ZM)FlF#$Xha*5!wKv*F2 zAGt5}1NshbhZ85+na?KH!R7LsqkE?ot0(W_`V%{uErv&lPr)@f@GpV0ytf~>2*r~I zy_KNy`ZKoH4$#kbf?Ucni)N}N5%+WD7k>YZuBS`M@C<%NZR&f%tYm)|8}CDB zE(4OmEqKWN8*2WDfKNs8@Ow%bjan2)TlAJ;;iDO#npsS@vn>0P<_&mi^FoYNRt7u8 z3cUDs7RhT4#*%?#2!Hd5U{5uU?odO!Gv*NS>=}fuRwf(H9|7@7U1C}rKvFddkZJP4 z?S;C~x5MT;Dq)IjT#5@11UbZz2W@g#qUq>%68nOXulO!K~ zN-I-s>CTUAPQ{>^1PI4q)SMA=eT3z?DEwiDrB>o;_M9oSyb~U4)zYAMF%aAQ9i|zu z9Eo~s*e-k;UX|sryMJx++U+1sU)2l0Rn&;SxHs(EcALKQvcbVw^ROznmkJc>W6Bar zTQoM4v%PG8h3zjcGE0D|T5NazP!ar>#qvfZ9vNh;FlDlp58xpaD|B#TyXP0Sqi$IN zE(wu`lAa^9**^`XJX4VGP?G+Gy+^>l^9%K9Ri-<=f}tzk9s|J#_+~@(+DCZ?$^+@> zlE{Y=B8jl?yC$c$M!Jw^LzMcDW3z$~Q@?B(;b(nUe9YsjFK>5MNAB@p z-2~R)sG*G|lV`Bzr~vkzenzfoMZp)x4UE{4DJ*|15PN6#ldV>^uzeyIyf-Z;2faN( zS@8hswQK^u*Dt8?Pzo8(7KJqfz7TUo5baWCfb_OhGNyF}`8pJ;Gu@Mkjsd$@GXKDF zSZfW9AEq$wS3c2loq2e77MuJ2bOMjtYJ=EAe~7nY|3*9?Zu@P+qa!i+Rf_H0r-U=! zi;EeXoP+o((T`mAD_}FMS@gHxPCC8uIki8(5bKW}CV4sgG5hW^4j+4NEB~pEe>tt9S4_1Ih-zYuyEENU%{poO49=d9KK%rYIMyvRf_6Oo%KSc&64{<5o6#*SK zd-2rS40zue4Ts&|l6kAyK8(y4@_#mM_+4GF^(mlx8aeQ<`97FdC6Vcpa_FR%MdWA% zNx0ul`1C%}%Xe?V#s#HJ4Ahazb&sj^+i9G)2fV=Oju6Z(T0vyzh9TE03FgGj?fiGY8AKQS@9#965}WI3&tXTgPJI`1M#i zXRZQN1yi`DRRNEFYg5~k3vsY;Kb(obfUZAdh_c&Wyww=Yx&7aCG;WKafBs72{BuiS zxz85(@pX_hi#Wiv*V*VPFM;!K*u$@PQfn{L;zIwGA1eO>>V@fPYj!tKHQ37ZkH$jNY5% z;9*b@vTz)#dipwiWc}4k9QMQ4?@LjV-Px>rwgX3`H^E^|Z&bXR&3cnhVh@}Bt+{%c zSQQ20o+ZjyP;P=@l?0mPA|a?)8x|cm!Ci0UV2|c>sQ={$Wuivdnw>&?{#?YH+j78k z`2hJnAxd73jZw$?t8i+>hg8mE@Z5bQiw}uH+?W9dZdi!Z+S%V~;AN87zk_URtb*O! zLg~hc0*E^lk0mx{)dSv9th@I)S*m!&!2JACw0@h5AFZ`uefwT=x=Mkr7EOVrC!TYh z zb_9N`YJ#-e{G{skKfKnZ!R`@Rs*T2GDi|^cZYrF^-iY0FX>}7#_Q;2i=8~`{;3MRV zZ^NW65m@kQCnpbD=-jg+V7s9JQa2xBwAWd z1Y}-k!>_h$_$qLm{8$))I~L5O4*~?>hT98JzT}6u6@*DsyCRj9e?`okEJ4n^2sdqM zLEo0E9LYDV^ZdURz?C=*?Y4_C`NR`4L&%1aRnDQUUN31;!xm_0UI3$w|54?QOEG`r z6wK3V0OgS&;%)teQ@%(Q#j_N^&CU7f#++;5WIm1bh3|yMyJum))_39{3g|xU&r3$C`hL{?~7pP>SYkV8(~ZH z9n7cUsQWGwD!#4)sjgXI{a+i$VYMh+c-T#n+U4Nb4O5)HEe>D4eFZZ+*!ledAy}3e zOV!?_!<)qIa3|pe4rLT^#Q)BM>YCqGQq|%ZDxrllDwbkFDIfmP1zf&%CSF-^jVaXG z3eCHbUjDUG7yu`&EUEG;sfPY9?^DtPWSYrqvx`R zkdTHp!RxGtW0~3yFa9`IN zY;Px%{pks?NtV6KtUb#1xsO4vq%z#1eavb8CccO7c0mnrSK=WH=|eAW+KAZKUs`>Wfxz7rn@M8XknA z&q<)*8-afBS;k-Q8<4n?3TG~RaX!5+qP6?fp&@#!zWEOcp3nMayygXdq>JUHF7MPP z*COKJbG{#>ynPEt^+w3bZC|L-f?~)K`$yRQKK$x=N7u3Y-TO%%=v}*mM&0O#=`f2^ zy5|J$70j(3?9c$Jlm{DHJg8e;DNzv<$6b2_dBcT$xOwpkYQ4S$?HVg7V-$mN#~%>6 zeM!*ncOBzgb;)s+L$DK8qovYmxLc}8;b%Lb{sOdHK*4C8dBOl-Xh4oYA z&{dhKp7U9ncS28rxZjl|XOuQEnX^Zr#JY{6crXJ~+~?w(UwQa}J?GVF_oC3Q#UQmS zku036gA<$rjL21lpM#lb6Sx7oOO?3RFW;j5qJNc<4V|#XaVz;6o)G^ z1hH!xFuv_CC{EqT%iJ1|C&o4t#k(mWo0m^K9d4181A};8(wa!-Ome0j4y5-yrqH1* zEqLae&P=m9lay|(#T`wH{jjOJEtNpQzOq52$&$jN>^Pf-T;&Kt4eQcLYV!^kXsj za;rC7%`;_Mi>?y#Rh{tU8L)g3z%mUR!GFB~H_MHZJFExFPgW45FYM>MpJ<2S%+=iY2$5{!bzD_0ZbupvT_XU?1tOK*V zHOw#nR(cwCph$re&2<+c{4?FbMLz5P#Xgg0vzLtX}h;=|o^!|7ijbKpGWn2dp6a5kz9 zy+-@cX}nu)VaQw950x!RF#hN-IovuI?xaS*u1X)U$lS~0uWqL5&(<+Q-}x~&;vlHX zj1!AaNtCQFZ*aa&t2=TrqCE)4f z+o~N_^%JkZg=D5Y`^>)3h%z@f<8#r)oO57V-Edn1E}gOmOXth5KNI25`T+xvy{+_S zggq_{y-T)DekFz8bC7x8Pec6Y;#pR3mK7A?tJzvGdYz@E;kJ(@}&Y{D#At>k&4_@(7a0qsZvR=cprikx5Ea!=-BJWY6<1rrN?DeQ(EszeOWC@wt!V zuM|P75Bam7Z563?3j&+<*RTp#lI{n0fVfu?l3YNe^~88}sW)(R-ZL>yISq>f*Q2JV6nB1X4w%d*xGnl0x#d~G<|zEZ zVq+2*#ja#z&FoM$Za@ChF^8{vX25F4IJmD-PO?5XKw^_QwAwoIWGsGz>Ak(U&V>Wk ztQ+R?SP9K%^VI#8KWNJLAGAAhE-pSBPHF8JYzjThF&2yFY#9e!cZ7i7t5>wnToE?T zEQ5=_*$~rHN*R*D8V(6@IZAc&Y69e`NzlRI*uhnf;P*J zNVou7hAXIbVHJ%V?V&Tonn6A{0H)fBafi=la6|>1@sL>x#;K=}O!XvORuqZV+MQ(g zW-nOv=rMTdb9?;j}o~B$-CDehT2n6Jm_X z`q{9i`VC#@5lnXemx=!r_n@y(C{`M+rW!A1!uP(1v`bHp`}nIbXJGmP-bv5baMkrC zXZ%?txiJ)jjSo#&7Uph5lPoed#1=J6^x&uZT=vW<4Sg1E#QXPSB6YN!vu)uP;AGuL zt9glV^ulY7_Qfd}-O<4K`=6?dGDRInjuv7z zUjQ(r`yqBBkEp!ygm}p*pyJ_5^VZ37+jpB&MpKOYIwFN;c80@UA4mLqvIgo~9?<(W z!MOeX9p*+=83cbwW%kQa8k!!?nN}hWHvLz?Y^;X&W_3g1(!T1O-&^p)iBPA@I*X%^l9S(xA{|sQ&n&nh%%_mNDZ3-Q|@R6ez z<^Y!k_CdWA7Y{$PhAPu+kWM%bQIL-p%s!d-A{DP=X3%nkaMHM_W;qEFq1dS-xUqtZM zzu8c9RD#zKE<}|RZ_tqC>~r6~1xs3!U_>z$hsJJz%l=44d=3YGzfZ$`4ubIeOaNHx z>fvWdhnkLUV0q*h{o@?KtRLTjjZ0L3SRNqvv_hcS+=QN8Y>1X$gGqVZ3~uR%EJmSL zj;Aq4jYj-#$6?V2V0E!Z|A**QZZjUDnv5{lO71(AAM+>YWaiV)KE9ASv7Ym|wgRh- ztZ;|yMv&V35(Ccer;i68f{Wu++8LP)({8d{8s;4os8(_;+bAhf{Q&MdT!`J20iB=v z*?!;*|KkhA60g2h7 zVCs>`DXd$H0dA2{bZQCyTayfbABbT7oz-;7*TeMM*_|xkPnOLL7{d6}agx_MjaJ)* zkUi@yaO7JMYO0*aKB9-3n%5wp=`oJaoFwK8#wrQ#@7>Uw>SS0$&M7oo+ra(D$RjE zxuz&MX~kP&+k+9WDzTWmhi+c548khXV8d=-cz?DE?9OK3>ZhJ$%A_Yz>wikpiqoL& zp9L0$Im51l3~EFj#52YsP;`;~e&oENyAH;XtD6_2w?QV1pNj$i&=Q!LT*}FJRf2wl z`$WP)8Gl;jLF3;;puV)8Ns=|few}~hs$)0J`_K=;$D?sh-3@qsXdm@jpF`yLr$NM> zbV%A!Pb3RRXc608t*GxKo<>{Xmu48)a0Q6zj0Bhv5QJY#(jk2;gV_2!WctPhi1B#< zY4;`U>~Rg@^Zdnj54$k%Y%_-Mss}G+F<#Ttd@$DTqd8}4U~k(sBBa22DN1~3jeHMm zw)set0!ui44%eYZcP1~Kw-RS(0koeRRx3Q zF$f!&iiYvWKxsDzGnDwbL+hQ{`Hu{Ec=4#<&$FB`3nRL}crMGAR)Vz9JOd}8FR;A( z9vo*`X`x@)d)DxD9DB7Kj~#C$m&z~0=yeJ5u)+s=3Il;_Kf?Mz8>rOcMrP9M1qrkl#{9xU)nPosX} zr!daO@p!r*YzPwCY^a3NEAq-4;GpMDs15MPg#G&XS8qA&pVbNmcF(D{w=9YYE6_Xh zkaeuCWORNfKznQ;{1INlyC@h*Y~{}24bNThob6+U%zq0CUjN9u;$H9fhsyaM?9(ehIN7F z;hw`)bpDb(7=5yX^Q5+|DyhBzeAAt2_)Seb+lJg(kM(@F|)h<6E2T&l$74*vL0&6Jv~Qsjw@8R0Cs0&E|21ow-4 zbUg1Y)T@?|khuZyn7wCz;g`lYv)V{zlNvA3p_uAw9fnHtlNk4D5gLDX!LE}lX|JCK zri%rD@5oW=G;#t}w_NIZ%$v(QX__u!kwNhBS$D5w> z*opmJ+B7lS1NLj)q3Jp1@U;0DruP<7i(e5ScqAXcjwE5i^a0vrB8w4jB6!T+6Y(e>WMs+0Q!L*Bn%Tl+Zr=*<`X{FTBk6BdeT0V~MvcQTdZXJ~_*ik9Rvc>z*gT z=%?>=xv)R9E)KzID;#m5^i=5O@1a#T45%6>qYA$fiszPruH{7|_r zln;!9H^QB5?Ebl8Ct7Qr!}E7ba8{%n4xC)h8%~mftJ#TY<&yzXuiw$AN;fcL8Gy?^ z22=YkSzNoXhZ;5OqsgEmT#r3K_5==5{oJ$YvArCaV;|_3mV3B;ia&n(#e)Kp4|BVM zz~o{KXlyotKQszTRyx7@Y1vHk<79M7Wq1EN4?$<48B8`jr+vM7R9D`S-sn9}(z~U= z-L{!Dp16j!ucas{NQ67*_(-a;h{5CV*OXf|jXS@+(_qNBQD1tt0xY`vRln~+JUupU zf(NJ`)hew3OJ#e!aPAhKj(xy6FYe3M9Odv0{|fYf^^Xqp$CBTZ$+Y4j`@ejiiV;)= zg*5h)q3e%{t>Yok6Pkg`11G@a$~`i30efze+Je<)Riw<=7lSp^AuqQJ>#EaGcpe9C zaXTRD?lp8Ytr8JnSV&K6u| z%E8I}Sai-k3muX>G1KER9(vPEx8A#r7xc4mpLGm2ud3sWuD;D-iVNsBIYbu)Jy<=e z!d)Gl8*pIpdSa-10FUmuOFTmbhy!Pk!0Try7nTM|lL>hHkTfXrzsA@10n#N@ zKyt@9Jf*w~PN}bk17coSEIbnu#SX)Tna5eq^gOWl?IG$rB_J}U8AgTP8%U;9M z=Ui!er$7nXWW|_+wL3|P%u_PLVZToq39Q4|1helD9JOU#M33VbPR#<|x_C2mu6ta4 zUZMgXoMhc#|Bd3keOK^nsU$b-`+V-TGZOf2rwB|G>|qkE+i}&w3z%-1j`4~V!#ntK z?#3lhzAlCoI6cM8l;tRD)k2;wEu?CO zIH&VfIGqXdWUKi@+#30ZIXUA$4#=|k&4yI+iEd(sQ!k*I;cTpY{|k=@`Y@^qE2wUq z9>=6(5gct&=h@aR<%UcDtuVbTi}P7o}n0uaAg7CYIRF%9B}{81s$I%|g2 z!CCOg`VMiK{sv!NGQcS{@*RghaZc!gYASQR*XhK)W`&pDYM19IVVAF;25sD zGX<{fQG~i+ZQ%N{?rDAj^p9q_EyqVdY1bzbk?D(8Td!9i9K8V@>v!Y5gXvh;F_(sg zY=eNhI56rRLCL`kn77y!EZFaucCR?NJ%0_eR~cbWo)dgFR01jXojsGy8rxg%;n^s> z#xrZ;;W}f3|GZk6dg)VS%}r!}PP`(UkAX*_zsQT0#(jodOU%G97|#Fl|(7@JWYD#IV2`Wbo0UoddS856I`VDFc5&bSBb z%Vqa$e^G^4zAwg(zkzu0&v#Hh7eRM>J7D2)p=wHmMv-H|O9Ebsh{nzW~EfN@An}$vMqt z5?6i{9u?hz=)F^U6Z6;NmN0+3u&AFbN;-_;>WbC6=ksZ=P9_<$&_jogAk2GWz>!u+ z$Dx5QNan`j{LFSZ(Op+9;W`WVugJl((;A?JPZXoxQiDQ?OYHd}sybRF4H=U`tS7P< z`Lh_Wl*FShR}cK76ghuf#d(4&uX3VxoUPu!?jtfAvvC9KXZG}7h@s|2@I2xW46;4A zfOl+9(A@xf*dB&y!VNNGjwmnY?jM-3avjQz*V4kTHU@Tv3#dszFv>j*LGI{#RClzd z55KXDCfi-`GDjZ9B?P!_H8Xjd(R^H;K`~zOK_8+|)uF^Y2+uT)l7*)?fyMAnNQ`o& zTHT+>h;j}t*S$j(+9c=-e^vaN?@RVMmLcg0MV&V@z*E8m_3fs@iL375BiR8np6o}# zfjZ*cKR~{wHsiO1BarD?1^?sdOx&sbz9>$HN=S*KL^G8krE=f1FEUhU(nN%cQiL=s zWuE7mj48>Sq_}6_P((#&QW_LVeH%2>!0$Z2KY-^xm-jtq@3q!viTf~qjD`w)GjArJ z$6bZx3erNExmkp22*`t1EjYO>hYtT_{R>6Aag{^}xCGTvLv3HWV3_TJmNawzTO?t5 zqA#3MafF;U4H#LukSGp_VuP6|1P=-E@2&fc>2UyUQe0`X$t*}T5u(=p%b2jH5M{3G z@xj&$vr}T={h=hNs1}1({tK${ z>(N?oC(0~x0O^x^xs{)e;vKbM+{p6kslygzf6)SZT6PhN%~Ay2?nqFX_!!HdD8lMp z8R(Ir0198&ESegz{PMUWurg*{$ukK}VeF>g2fT6AxdUjc{0BceEh6&2-lO{Td`t*u zy+lE(IClR_IHlVukbY-IuHY^@mOQ4ME7SPg)%YzwCFDl(ORVge#1CUkJH?GFXnI~5 z${5auJTX5ie|M(vQ1@y~W;0EsM+mLzc!%AlOYxFr0Nahm3W`P?!9zw-Sg3svQax{B zo_M%mp%m*mI&%_FiRHsW*owLrddY9EMld)iCR|zlh*~NeqMK(dn#32=Et-$8A^$li zXwJoAZC?o8)`~it-DtVyEKG{rPA>GcpuyDZ7|s4hU9UAsj`OLpblFlQKW;pZOwe!J?*g=Zr z4Y<`v=g=f?g62 zc(bm|rqhGhki%caV6DVO+@r)N=O$hU{e*6iC{`AZc(cw`l~QCqD-uB?B%6kV5E@SNq5TF=!G+zI|GFqgN+9*?8=YhN*jtXGAk$z9yMu=CXGzt!03 z+*;`HMBj( zmc%<3lCsyo!MV{4r`sMQ>rKYu?&)eY4U;myq5RZ(=@v zHbIR)oM2lZ7*4Xml2&cFVYL%7PBDIRkUPX}?}VHgpV3ffm}Ew#LCROeqk3wv;GYic zvKqh>Kl2HX`4AV%8{<%~7+kfO1FNEVxGcdOL`;?8*LQmi?%GP$-|%C+*~|3$#sg4g z{1{rkRiNgxW_sB|4qqNs!=+_X!rZzOV8$3GT8)WZio7%4E4oH>w8X&k1kyKWgP2EU zv~b6Xwi+*k0_r?*EgFAe5tO)Eo&n7V-LYp-BE+6^>zt3S0h547rMQ>vA()xSk&_b! zs73D?QhseaCwurkYW=ij5S_KOS#tq?s$>1YKN&B^qY1;yoFR?*pQUvPT>rTX$4)E3 zvl-{ew2NZ6yI~Kh???qXKgL4wl_S$64#HfqL3o)g1AgYSVVuESxHw@F{7Jq9W0ux{ zy4*A{n0oO<5d^JjpF2_X;(sX*HCpA=LJd={O@YU%67KpP<<0MOP-@TNC^tzzz z=~SwvB7>rR>(F3(7W&1t;x^t6LC(w_80dCe5W6a$cXGQMy2Z)ifuOO_Z!%1Hc_ujd zdKDQf8$|6H*9}|CNzD2kcxP9KAXwC15YouXQfsRL9s zB;vR900=KMgF#bC2+2(&(7GGeETizFky5XcNVs;^gx7Og6qQ3n;6d0R*_F2lj7RDH z-%Fj4+(1v%f2n!8RfMl_I1pS!Wb8vFw{Dh;U6YS~ zr!Irc!}}yR&mD-h5{@?bK_8XW5ZjMc0{5j0aqW8_JT}__oa;;lvsK4|%5x#7cjh9} zX+Ci3fE(_6e1#TIt|!hC=g13|8%XAY(UWr1&7vHty=X4dEoe^16 zA}}T`_1m%4U?s8jE`>xhU)cVE&CQkJSebC09FzP`lVuix+LCz!mGT8RD#;CMU(dxN zO?APwS3!8oFP-LJ&4fkT%c-FZ$8}3t36`?wQgW9Q&U9_3_IJXtLwP#xr#m3h{V+V+ zxd%oCNWs#l9klO33>jlkPHJUlz&Y&zywmfQDBn7WT!1pX_8J3Ox6dJ+{*dsbJL$7; zHFPZBm+{NAIIo&%V5xKgC3mX8q5e>MW_lD%8DQR<1_xAp;ZEmmia=kfRlMt4XW+ig ziO_ap4xKgrZ?*k~C#;Jzka=UmKrXz6iafS~m&b-k`b%3p7T-&bJaWKWO160Lx+T*H z*CIQoNCD3gzu&n>59uzW;w$!o`^x}0y{(&!);$lA!Al`^X9`HiTEM9*oz!v3Sn8P` zLoPWof456CJT5v*nmjyU)%>TVc1$vFczm<@(kxBaf^l?H3pS~StzF~P938U5YoOK3%+ikCdzM!THyyl z1P5w-OH62Gj)6Nc+`%^F9SlG104Q4C(;Httya5rx%COSt!ldU=UDpw1f zb|r%NIX9R-G!{f+O|DsN&qn*o1^A&f73LcE(f(h2Ol^xJ{Uh^fPA=O+^u=(ci<9Al zbQhJZ3Wh-K1l-vc0QK{Nq3!hy=&%R|IpY}o6q*g9?E9>2c}#2fj-|4P)1l+cM2wZX z$~A?g;!SNktQBGk4oW}SV=i&1h30yLG5~>u^ z@B?29tWT*?X?<_VF__8qA60}%#b9{)I118FmQkyW1Nc)n2E0P&z^ptDC!LFjb9Qm` zNY6JKdh90k^%L@zzsUo&*4dmr{|>c2)IeSpOvjwLn*rFcrU^nuH&^S$Ed6csj8?1d1zUnNRr{d7$hJn~L?| z#U*R3jaW1wiIA7aVw#M4mJ6+BIw=x}lahc3uuT zuAVI@O`VNz-IkzE!dA2`*T#WW;l#VNh?+bOr@y?-uyUgn6wmCZj_c=g$2=m)i#Iu7 zUFwY)39rcOH$(K40%L=07s2}h2Q>V&mkgBrAhBb6=t;Kw&nnr3zjH#MEjpVX z|CLFXn=XKd8J;r9*c7YpT00K8_c8!+#b*iNw);;Fa5w_!Azlt^hIdmxxfm+IPjSoLY@>{g6cH3 zo1XPmFflz6Zi@7if(tuQ<2OgBl>vBIYC@>yO*;8R7Op?&LE3xE$cYLiTx-t9u02z+ z5$%{arj?7Dqk(F2_Uva5VzAtFNEQzPnll&t>0(%>eg)M+B%n_t6^|XBf%hgzU`M13 zUSF$%`92SsKP((vWqonZ^0Ck~pUuy=E|BA*56SKJZuk*&51uRx#nC&f;q)XW61r9$ zOQuTk*Ro8>8l$yX{rMIw$ox$LGv#4i{s4LLWRUzEBaYpZM`68nI{1XALdxY%2+aXWKx{ov9$-P){EeM1zmWE9Uhqqpo@ZROH6vn%(dpW4mahIBh4U zQ!QEUq!bU_(B$H6j*$fg_et&JBs%$_3Qg)#BZ9B%$ws#4l-^tglZ<9!$NXLJ;d2|Q zzWEvt{&`4pB11t~)-5=tbryI#c96=|H_3;8l5k|6obZ^KCo#J83f4J1frDQ6v6p#0 zUljRbQ=2Lt&L0JG2HDX3LJ~h_K7!R9x9NAD5BIXC5DYppKz+w&!2UM+D5Dow+a}Pk zazk>yZVOsf_R_CESofIsO4zq266h#BH2rTeSzZ_k&Xcyml#|J*6FUt;tXYOzuZ7l= ziC`V;${4J9xV9k@>#j7?9Y-QzaHN$^PV#|*#xXQaQTCMQX|YYim4{{(Vx{NdzCDO8?WNVY6nP&2xD94=J# zfS!68njaJh-TTCZ$M-%!o~)=aQ@IL`J<>#jH;wrBya@mAo?TcuQcLTuD8V_mGBWfu z60~mRwfC(}nJ2>9=IMG|3#7-q?%vc@{WkeI~w9XeDvMqCztb zDWPGX6OGXA$B|D4P+4Zc#jdmj^@t8oiW&LqG6$I;(+M+66MDZsZO0ljo!4*Pz+ zsp_0S#A#iOjn&6l&K(bP_8Wq2tpFCWo&@==g*f-D2`p`TM2e1$;kySf0HelY&`~^( zrX{nX;YBU?$he%VKdS>1D(>Ot=u)(?d<|wxXTUy_aI|wcgA0#lk?&qP*JR{#q0yJI zzf6BKT{#h>HC?f9_&wC=PJmUZFA1a#lD}7auqBYfL1h(g&{iLo7$vaYZzq)2lM(La zMdMT*<|V22VcE6Ge4C>QG}|VMcQ|<;Rj44OR`$XThmy6cX+vhbw{L?F%JV%zeoZCXROQvH{r6JJ|VgC!G$3hNcS(eqz#n4xWV9JjG z@_2fzpnBDBdQ{ttd;Yczm6jYK<4=fD-4)4r*u_f_8k9?yw?&}i(^MRvvJEN>npiGC zOlW#x4m#(1Q@xL^L}GFn_%7wa@N+3_cpL+1GbeEYQ(1S0b}5ZjoduW2?geWjb2xD) z1hutPxLFd@vet-o|3aKcR*^@(Nd=6phvkBR6PpG{OlOUP@V z2#jOiur{MAu$YtxT>NCPI=6|;h-I0yMmgfHEy*vn&*Tn2%Lh}{cUUoYI@)|^Jr9Wu z7_u)9TI1$`_MEcn?CM$Ib*={USJpwLK{wq!W<9D7)pCKOK5|3#eYBUg>(Fqpz_fz(&c-YyHz+IL*@oTG-K~w?=8l6aSu8F1C#i@_4%aya}_MNl}7!{ z3Fti%OhhXSxFj=ic%^)Tv(CvE%$J@a_@vZHe(^5D?RIu|&}kvxw-k^)wh7Q(Vh9Dy zKj(OI7L?7(#Pg@$vkZ)wu=U<_41X#jm>h4$ys)!z*v$=k4JP2>=R%CjV{>JH82+ry zhFK59`Td)FNPB1ozGOWeGAI1-#1aGAH*P$??8*bGyf~dx9BF`|=pM|xb*EZkb}2RZ zZycEZZU9p*27c$Bg6?=FVOP;)p`r=J%N;=&y>ptd_)QRLvq>vv!ir_zB*Xg1__drf9v<}6~oy} z8gNKqAAWu}23?LXhMV8jaL2xVXxXod*%AWcIX_V#co+j0LM4T+{Znw;Tv2{+;$7M? z*$=WVi$k^gQXI2Now$n1L6O1@ZhJ>CY|%rydS)$2xBnrSHxLe^3ddr;eHK*~aRdqT zT+;HGc_bA+k!|EBiEuwiwH7?Z-?`SPcQb}e_hPx7aUqbN_<`_0>Z8S_K|1N-5J&9n zAvZM$RhLPj){J;uQ7wyyH%h{y<=M=WAq`>feuA1MQk=t?>BM+)IG9KZp}OuF&AS>1 zwFzfom)1wZXV2l9t)l#kJ)xi%W{1&DF(~-825kGDHjlRgIrE9^*95H_$~}3!sAw!_kQ+aP(gG9^D&H)f?3?*8CfF4ptEC7>K4y zxEpWFD}pn7w}y=b;Zx61U=hY6PnlPAFKwp^JZ*4N9pJ>ho)e#3=G&F5rH)A)j9PLZ zK4-X5+2HL^{CNxZ>nv;EC zbaER=Up$Iuwq1j@oy9fRm{;|_r5&kp&gVqCF4M2`Oo)f2AIxu(z=Q1v@q^M0*wA#B z{EI07!69>a@v)gIsc2x?2UC!J5{s{;^@V|Fuj9By`$*CHvw~j_=Lu8KoS})am#O>b z4d8REmwcZ2hD4i6lW({T;se5Aqkbeen7k6M{Y(aV*%)dpw;ocHH;|FvW)KwB4@vJ> zevHi|5zKGd`9cTW?`HCxf2xvUorQQ}MH!i$)dR;XBVlE+ywFV02v>~u;p-7KoWJG` z990RTvF$%;z@r45lluad(raPgM18WwfI_qB5>V`zgwmhRfkb{Ouj<_mdhw6}efWtn z>oe78sD>(iSS=)Z&;0Q9qaITDT@lSqbm)dIMYM(}FwbE<^*jNT_EnIR#oLIHk{#5?|{GXI`8t{9njVC1Oj=NsO9o0F#OsH^S>sO#Ysbub8Rn{IETW#vBotg z&kxa!W3s4gILpXP-GCMs4{|4r#xY(}C^j?EpyM^5iXMqm5c{OM% zcqI=puIl{jjU`7pDA>5{F7 zz^pj~uKC@j9={T}wwl}kV3d$H5R_FoGl-v;FU`3P6D8_8`oD-z9g z;T<3E(91z91)2eN@NzHH6MMs9PSYEr)9XYO{@tP3TNi@;3u%Z}^hK3HCElW)%x4`U z3%oK9lxna8lXov5&HV`7pOwtrw|oVs_;FxXZ~<>9nW0vK3ch98B{To|Ffu=!7E4w` zqC7iWj4l(!_OJB9Z$>Qbj?27NaG%;OZcx!5q72!c%GMfQ9S9YO zcoow96D!EjrxtE?$`Y)U;C3gyY>&2mpeZ+V2Z|L>;e@|^Sg?idBnsXNQW?u?*rXUH zJMD!2%r-K0c{=sq^$&DfBj|}0v-!;Z4^sY9`7;#!{Q7RIO!G6nzluSnN(Q?kl=6scG1gSXGjNa;mo{@0*psGs-?efGwnRiF*o z-Qo{jQzV5M{B{g?oCI~eWPOaK8-+HZ$q! z5tb*nJP(TMO?w;xlriWc%)(1ISc;z<)En5k8$E5`8 zBl_^okA7SwYYcgPC0O0?gPwdR2ZplBd%*4fu zs##xO0vwLp3~ozp1q~+Y_@-2k4$Aoo&ePkV;kS+2`K$-62DXFHo52tCpTgboUP6u~ zi=%y^4W0jMH_kmCir+Jy;7uZq4^&3Toc_tg@Y@6sEj^2+>%P!00UB6)KZ>Lu8$}Pe z9tYEZQlPQg4T!on9LV_%TiR!l{7wgmd0&FN=lMf`O(A}_8Ota8cx2;273i901mogX z!qtsu==cZ$X2)mY&i2!=WkmwE%e;i^4- zlH~8hwcIg6-K!u#zba^L6A>m%3_?z`5yjSAcTDC)o z_&9WQ8V2pzuj!p4BsGN+D~npTo?sf=?l90E0~w5ememQXw@0|y>SV|mXO z@Ltslp?|Y5&-*fcdFle|DEUl&1l)&@pR&Qf+JO6&wv%F}^MoBC#$izvfKZ8|I1bP}pgN;iVsY4(c7eio~e;ySu zAIzZ@Y2ZFwPud#Sf%DkoFu>1(1c{Acl`TuH8_WNpzbP4C@@BUNI@SwS7L?j>(0czjs6NOf*Qs3Qf0Gac%taZACbFcp-KfW~* z$64#h2G>bsV{jSpTNjc~3x%K|zZ9+n+R)aBQJnTINxc8$4kmi-g#8K<=q-AX&T|@} zy!%hUY`Z6#u;=9MUJZ0rwt!y+uVEw0%l&jmtZSTu@9ov`YRG4t8QVaoFK4|4*CNS> zENLoVt_R)2N^minu_X@F(j(rlSl4qJNfBm2C(F-|SpOt3Qqxg)$pfmJ^ptpP3W9~2 zVK8RvB;lhawO~C|20NZTCgz>HY06JS8c^B@^JCBB36Cw%ra2Dwta%Hkr&NN-fHUrQ z@CC!umH0JPA7ajEK!0vME{~c?>v{3;2|6KS`CoxpfGxcK%U%gS3t|4wKLWo^h2-<& zNg(@H24i)q$mEY!D7)c0d7w}X*49Qfo*$Ia$D9u?b0>n#si|1Y_!F1*7|>DU3NSra zg#XXUQ^2Jxt+`oWg(}|fpd<7Hs?VEW00Q|$}$K2bhGqU;fr09$v?>yT5DzuL%J+O?&n8# zji}IhUY7Vg%$?egyrtq6qNLYy1k%#{$PU$i#JBtz=~;0B&d1K@#9goBP4~Mfk8>eF@B?Tj=5HRk*t{ zM1X0Vus}o$&PjKm_Qzl-82kj=43oJ_JMNHIK5p=6^;+t2P(;{O1^A4bz`?~v5cWYG zzv-OBm>xUazik|U?B4>k)$oC8tA%vc-xq?zU#mEMPcu?`P=v&@8E0F%oN$N9UvjhI zIEE;{r4v{Epxs9w5Q%G2kluL>o_$o|J7f>A47X71@=M}cYQjNf#&&zP+Ic1;hVg zs& zCfh|2rQ6hvb;w0etD*M-x8lCD?AbB5XFO;(&@$W&o|oA>GfJI|UJ`|}oFQ2m!YB$Y z{^+h%2gX_pAkcvIfS!~RE<4vnH)@x`36l;Idet2de_2X37|+=3mx1tKq$Qk5O~qIC z&Uo=XdroF}(U9PM5Hd6$y2Oe|M9NokUuHji*tnZ@lpKM(?Q>yE?p?aDYB{fZ`6%J^ zOOr@_Km=-KUk9JGi#S0p9hYYv#;Kbx!%^4W5UKwWRloI+!_O8&p}9DDdV>g`##|<; z-c5opSBpsdp+dpbQyT>8%+DM#)fj5ne67pHk$+!%xlL07s7CoKXtm_QbNf)p`*Z<% zNZ|6Q93|By09vg*#Dd#H8V}Kt3}T3q#VmZGj zk55DCx^#MPyDdcLUL@1^djMBvgB7VQ%o|vbdy88I5?21W)^#=73NJplHLi)mNf!0n1{?o=qLM^tOJ9;q;KCe6n@kL)bdDU2aHpY$M z(?;yh&fsExbzOb^M<%+U=P)b4{o=6x&$&2eS zT5&1rr6z&no0-CDO>>&fFDDbH_2RF`kMYjPWtb6qg=q_S1Zl2%g2?BS`3;MHlI5!G zzOjLwqcVy3a=)lB?aL5R(9a^}PY)B3dz*R1mSX&8ZSgpNeJNSE$^(v=t)y3GT!obr zL$FDz9r^c?iJ7N5-8w-UOj~E+8iQ04S+|v(^qR+y?_7Z6yIQ!ULmOb3X%Y0i7|WMg zz8%v;5=mOy6Y8~d8jTod2oeX+p!4J?8qRh)*L;33zp5(kC{m)8zA<#U?L`}`A zu7jUm3p|W`jrVnjA?o85I(2F_^8v0VXBJly5#3m_BhZ(u+men}o{I2-WpOU6J7dB& zH&~MOm=>KF#&Oy#w^=ldyPUoV&fQl7Ze1gWIPYW~B}dr(wS?L{l!1Q>Qt;`J57}~{ zgk`!nfn#DYDY1M^m&Yceg4TI%S$GoZFqZ=z1sS2*!eKi0yskCPU^4FNZ-`7b2INT zfBfQiu+^?eV5}a^ePQ=%Z7=cv3e-5EgcT-mbk4!CX1r z9F_t9%=@_&kMaq3eJPHpohLd{K18qbCVu#>1KYop1g3Z!Fq1MVOEoESUM@H1_*t;~|GAfptnQde>i}p?5E$S6CSt4n9u0 zSikQ9ro)<#-UVi{l=_J(VMqeg^m5QK5c`eDpsRJ(N2=DKbu@?U4@f8jBtN# zF`c?M6&!EBqN*2tDTYcw=k9h;cKZ(}mb8QYp<+4^X9%CWB~X8pJpbZIA#Io)NhQ*+V9=YfSn#)5FurNGAZ)0Ds@!M#Jp2DPd0j>R zWU-Hz>xiO;pL1>*?`MHK3~bts(bBmTpKoNn zD@IgGjia$28K+@*m|X2$jZ57#=vqnMWs6H}x zW;RZV49CclZE!fy4de?Ru<^*BGXxnW(AC-38~Ga1!QfD z7A(2ahNtD$;{M0aprlz7R{Yror=-hJVPHMn2uZhhHw&}2B4&(OCu7KhvS7_^}Fy8eU7ND;96qihm#;g82$vazp znpcnpSL$==&rR{@9Ggf@M;1|~u`J^-D-t@_gusG(a*$Xy9)i|Cf_0DlhnF7l-l0Q%UdGagGFkgga2mS6* zwcpxQzQh_W>)&(RtMzf;h(CmEG{vKGLY&bh3LZ{1=vKgC*d-BklI};(%9$XsXN|y6 zn8aIKbQU9@s^fOuM$)zTH!ZEugLW5jIC-KHT&s$xN_aGPy)2C02vj4}?o|*usc1;H z8iU(+-Q@I3uc5F0Qc}L_4|#CnhCuU0CRGc1L?c?9x!tGF0;!r0y^`kz$0Y9v4!C=x z$Ey~>>a_>4*)EW7-`Y-4Jqpx@waAgTlhG?QkTDW}!I1A+!Ggjv?f|<}#a8BHv$#BI z&dmemih4Y5{}&$IR>S>^H^ZUp@vtiH8sNSJ%vHHc^$*sOg*zf~WSXjVTxhvZg3_ix1d->4E|*2 zSFzoGv{ti&JtGxEd$q`%^EcqUe*s`>Cvi2ugqcU5;gw6Qhw{D>{i1Y; zyc+WwEMvZ%Olr73qvrI_wK$J?By>F3edv`Hj5lI=5C609Y0XZo+%^ij zS1#ssoi^f_8ykqX-&OjZbyNjT%f@Gq-x3}B<0NwU0IYvCmf9WJ2%Wy$;mEUF^lRlZ zbo<;*bqb=1&c}4n6^%g=2RDIQ=1)PB*K$bo3nNXEH|PhmIGR|l%e*79WWXm_aOms@ zYIeGUij>+2yqJ$O=i^*Fw#)@&l6T>c5l!5AUxR6oD^NWD-Ni}1%YpOl)yhz3C!Pb!qmRs#7S(B>NL-Wzu(V6>as*w z*m;@Mo=>WA>^lo%Gd7~jM+pdBagupF1iYxv(zx!l1sW<|7gRmS10G{2Op}i$k^hO} z*T;d_))x<#C(gscEs0>~(oAORZ^O_72guz|ND?P|fXgRaVyu>mUw)(`eer~r+pU3D zqgLP{_e->P%QTw0E)W{;G9E#$JJw#;!i+mXAeR(Cn;i+-kKCnYat2uS`z=j>@R0^| z0E#3f>W3_nVqLS-z{%?zvIr05SJbY_6NMM+ioAcaXOC$-{z9qP!F_rU_8x(j!a)VPXbQ3qyM!hRP}%_>SVLb z!EJTA+bJA-OV-2u(tL2$K7$1p++ma#%MmQdg;A|@8FSqqzb@L0ClXcB%Xb`pxcZv* z|Is4bR7V@yMz_$N80{F3W~q^A&AR0u^d@7~ zs6f_FwjTzPR$#1BmB5ni2&CT30G|LJCbWv6g3iemexF7xtkvj~8{`$MSjhTS~FIZAL=u7Fytmd0O7 zlkn*Bbr_iI4I&BlHPcMCfXMpWyj3l0NKWr+GWd8D3@CYlZ?PXY){ODcZxC!`JWI)^ z;n?kXnHyy*4R3~`&~U;sn3nQ~Oi^(MMO8^~sq)3AY}d80?K5LV=kgx*2jXycD=nV) zjMgnx!Vmok#Okpoo|IAnzCk`5Sy)N04$EU_j~w}l3mK*5i{ftbZSy$bblq&|l+?^~0Y?>-)Mt8=jK zj}^#F^cAcRh~^51Lj)n~j}o0szr=zSmtZ{IuH$>DY&s>F+y1l^5fb2V_zU@^`C{#F7|VhlF+0!lO{bZz$&qs z=*T|n=AAiMa>N6*Hd+z4t^k~=odNqT2e}nX9&*~hEbwk&D8AC^!u0-B64!DDULFj? zWgVgDUY1a^e_{-dUMm9q;f;{ibcWsv-32mB9H>}!6?PnJLj8^~SP&|SpQ} z;=F})?-wIj@WKy@K9%Cvk0KD)aRHW<6floYh+w-E#}zWamHPW8Zoko7)LeO;o@rZ; z8-iPa@7ze6mX@P~MIOn1{)c`@mO|yj@i1&wj!_yMlc$T|^zjnpq z^beOH=W(83)U@e19K94j2F9THuSR-zQXQnZWMFZXK3Lsn{s1lo%jJD>{mauZG;j`& zJ$i$WZ)ef)LQTj%5e#K#^5OUEa{O0x3?E(31Cx?95WQL!!X~$quXYVI%|3^QZ`+HV zKc<70&v#-!aD(NA^2s0LC>+zFijiNf$PLl+Xf|;pUa;{*ySM?Wds+gAr`XY#mw)4_ zzv@s^XAT;6-kj<42(s$Q6K>13BD_%+K~?%I1w9ic;2otL96f0ire0TN9SH_ROLYSO z!^^YaetIL07FDB%oTlPp{wHwQdK00igG?TM3l?3w19J^#pkv@U97TH}*KH<*YIoo( zn*@;d*o0SgA46}8Kb0NNC&GwMx^}BRsD3R+JEgC@WvQyd%QmSn|5!3hxQYsyk(sni zdyY$batWjUlSIvnRLWMK|1LZXB&EagOVxVlVXV27XirdVJ}1a@-opDEwTIUv8pJWUjOtWtdwuXc2&*Dt?Mmp);5`ni* zBQ9zDjx&|C`14ER=+8xKs7LNGm}heeE9?UB z7{tn$2(U9yzCTmA3L9;i9+cd;Fmi?H&Q9jfAInZd>1wi z>T@Qd-TiEgd{&40zqfk566)70sw*w*bq-7lQY2HRqr1 zfhO-58^KpvXusTzTJ_08>a9NDtY*>R#TD@K)k2nE-3xKojW}bkEl}oL!cFtcLLH4D zvggoiuu5zv^yn2hyV?%|RcpwrrGJRJtC%o}-GVL+1!3496`XKCSD-AG0uR=gqwLoO zC|NLy?{oG8CMS*)rVlXIh(H^UxRyf8oJdM8c49s$!F~UV(1Yut&T;R!jJ|%%pCAb} zXJ!gFcc$a(AKRE^W-4z&;UiSeT#8yXmGrlv0^g)93^IOL(u!FH2o9;FeaRA7dh{@P z&}z=tupNRZqu1odgl>FRZ2|6%6{Cf;X*=(CVj_AUsYlb)ulUw)Eu7imgfDMD1;sCmg*%Fx@SbowU%@30 z&h&^A@5(dyZiWe%E?7pb+p_T7gnV>*eGHR7i838@nAAzTpdksRFS3j9cR?oDtqaw6 zo^%_0_r$_=@5NBh^zt23EAZ0i_b}z$alz-+PX#7=cR~K@TJSJD&E~(C@M^y+kyr17 zVy`ZmuYZ(SsWp-2ptG=lXSU#Bg1^A>W;XOPUfaW;k^&>{6D=PJMg7&HOsVV#akd@& zy7)dWe(8=aqaK0S^WSvCpa@;MOoaa>sh-s3%!8u4VS-85dj#dajTm+4JVxHo!~OfJ z;KJOA!pc#MZT;me_0bm~F;#dL#BILAQ*F1N=BJ~=9rCe!aA zYu!bIV-@(Z#aED7GEr8+6T3HGz%0{MB<0n7cq?3q7F9FBF~kmCnxcp|PREI}B!pW$ zOTjPZG068^12TlFZE-VP>j7 zT7SPtcYZe}?+$LkZod&+R3yW15{$$hQJ+Au$p9;C-g9peXW`=Ri;Vi(IM^E?43D;7 zWA3{sGV>22gc}GVUX**$=7JB>hNN zRb{9un>0y^H!s1L_nsI-pm!11S9?njJcy^4jaXv3T#xI$?4)}y1cJ};?O?uf3*J%A zr;G3B(Ie$$q%}yD#;4z}vW+Boq&1MYxKA4AE8c>bvj+~n#aaDp*CC`%k0?`j*!)l#|7zbv_nYzL zk(4k_=lD5mj<)cMR3-Q(H z4Rr-DXJ<3?INyhF2M5uo%oy7Co~JXfhe58VJ7~+*61Oup*suP`qJKDrL>?EX$=fAi z$tyv=YUp{A$+0gw=2ehsAH2CN)i2VZyO(ZGzJS|eC^S!62W{KMShof-3-7RY5PJFy zj)j+?fL1su{Gmqms-Mtn9dm)0|HE~T|MA{L&p=_d`!E*XNLRcGp@%#ULUxoi+Y~aL z_00EylRZN0?5_Rn(c~bgUFHuadTRW7-ZeaP=PEsTYci-BY(UT!hoRO#uuu0HiGp0B znp}(Fj<(?MAOSZ^2O;IwL|kQg6Dx8~(+z7jQGtJ+C?j);gtuqF>BLFAa{Vk&aGHxX z_uZ-E%u!~I=>z=!;xs9gn#1<02GHMcHbbr7Gct1O8LcqAjGg{h!Mk1*6c5T#zAtyK zT8pxq64OD_fa75vU@U%h2cv_WAatEg0keKHDn3;XBY(uf!bLLhT5cWZet1oKLnh*f zr#3M0kS0#_n+***t|V#P5{}*BJd*p(;p}2ncs8Md2~)cXzik>o*;L=czy2L={>1Py zX7XvKXaW>&*a`a0+~+|xn*MmA3#XcQlS7)Nq;pnm)mlGsR@Es1j(TDl$@QcMVu{UeJGjCz`sTH)BOS*SZxJV)XniW+v z=g~O`)XhMPxO3D|VVsm{jo|9A0V-V3fv-Mt{nDR0u>2zfL1vua*q+NV*f50Ow3vv6 z=fI>nR3QWn!WZX6eR zg$}O0_YgCl1(J^s5F@89a*V};GZ~cH-DOYHpRz3GV zz68dTzQTyH85Do^&aaZh#+>`9yiLdV?4*L zIS$op-Lb;{KE6@vB6TkI@Lbw~Y`--DJ}Y=U3qO)+(5?SCY*r zcm@#;IdrPJ4n8_qhZQ3?c<@aZ(rQ(R(hO@bzP*gr#FnFUqCS6ccnPaBR!>L$&(Z5n zn^8K|9Hq{V2M7VzEa4m`yg@KEw3uyOTrQw5*nL)RR zs@-*Q(3S8IcTY+q-LJUL&f*8q$L)Ozo>3yv)W~zP+X#d1VR#|+FM8R{qnj0OTH+M1HuRCuKh!a8uSv8%ww=%^VQ6s>CwFv_7ZP&0%Kx_#<8XTq#RjfRZYRuFpZ9~BM_OaA@;PFC ze*vV%o<+xx8>wbmJ2NvZo08uyWQTYsZ&9WbG20bSj5OP@=v^v}k#z?4KtAv9;s|gL zn2ujL2F?ALIsBtj3dmCzMQk+Tn9Oe{^S2$Jz;>JZaNiGH_F$tAwK*&Rzh^7Kr*cKs zF{}!5U#x;^FDV{&89?IaS#W1e5r}Ol`SzuaxAx#_I#fRo($u-!f15b)w@PFA77cnb zC<8aoyUn!5Wl_TcArj_unm03HGa0d;Onfe0fSa=?u{c+m4F24PV?Xu5I7fl1&s&1q z;5zTdu~1%@MLNFw)&fUn+merY6WEgGT4)f>0s1I~2%5}-KijL3n8abe%^!^Ps)z4Y z1#oQAhl-HF86fia7|zI830vf&bLsRjdS2o)JkYT2T3qqeuAKCZ53)5XLf=cjLaN?cEu6cIM>mxxVVM7QMEmtQi z%z{B<_7je|-AUI>Y@|!I`rvbNH!sp;IxP;8gbm7`bniAUTT~YVyQjp!%(;B7KYkvP zejeeyjopSiqN05NU-v+*JA?XV`$5EifW=pfnDtZM(tB;Yv7u)voVz*1`zj`lmQ4b5 zG-oYZAF853?+)S5fLPMI7a||cm39jHC?v6e`I3KsR1$xvtk}hdOy6j9OCJ3g%>cJ$Ub|Mes zxcMU53t)Gp6Fk2>8S{jru|iz~ukoGWNS_3L(*8_!cTi+L)e+c>%jxWcL(MdbUYwJ>{)F&OD5kU#GJByL>_ ziJqy9(zEx&-I0xOEL$D2cXsoZHSZ)NrrG#L(TjJfpL3WpQ|W1+yCC|y6_)%O;T`J_ zCuc|9aA?Ct_~f!3C+qlM}H(9u3}o$I1Pwa&9j@in94*G~ieT{Cp^cgK_^Oz}d@EMw8F&{!@o*q~soccqdtc;ftxO;t zHs|2g(`4rFRvsj!@<8O#D%kgK8yvQlr$?hpKq78F^gb%22fSwE(oKe-Ho}8v(|OSO z!WSQ!l~9!h`Ve)u5|f+F;I2{;-dVpDUvycLnU-f^bNh7sy>tM}Kc1#D1#ZyaI@gg% zedi6KC~Puu1^b<0n4|QAG|#+*%y1x{l;yH_4tbm_R|axFZ9*NzvmnLq!LQfk$o}82 zNLIKb#Gkh{7Z1I`-2GNS633H3Z;d8CkSigf4Pp4^c@Ulcm4{@zLe($UiZQYfN1@C} z%9pxL=ZA6J?_H~Sv&J6d(VN1|4?$zNctsvxKk$UMWnbZ|@B&)R?JdH>S72qxO>Q^v zoY!^zGErs5&A=AzEP ze-M9U7D_mez~fV8Xz=+;l^Zt?#}>#_=}t%9AIRZ&2Ol`xX&$e|porQ}{XoT1_T$I7 z9+i7?`yo(~^H4g3@*bS5g=3@RI3n8$$>t?!y}bux-TI&>!<*!#hW_8>%t^;{RD^y|q}?^v?uRU~%0Z?e$4xROze55*2c zVfG(Snzbo8OSJ={$>bD4)}k_w=%=qDuLaCuN7X45``Jzx?LI^kgFCDC%5jjtUztRx zE03mq72$iEUIl#*ca)nqmwFGiV61`#+vU6k4D@_)`k{Leb}1G<41Qv?F87d6qOPE= zZ$N&C%!O(1T9};6{XF*o1Fo~q;<)A`&MPYc^Ijy8@$qYrb4(V`xTf=zEGDv#esUhO z!=wtfNTzQ!l|f8UhI+5nM0xczdi4ESyqmtBc7D>~$r)8KvmO{$`ChpT zRR;z6xWS)yq0}03%?oKR=M$NqcL48K1i}XCY2a-4>37*tMkBHdPBN%F}7Tr#toDF4o71l@PwqXcn&FgZu(i^}lR zoekJ`Lp}8G$#6J@tMILxAb+N5KJ8MFMCn{b^6Sh2*!JuU#{+M{!jQeZ&w?AsHL-2v z(D4!8UF!{?t)#-`rfbn^>vdRXISF(ab4azC!1hi!2>y0WL^?QyQ9Kb$(z!dASEU?; zBrU)*-wk1`(;pf=gYdx(J!YUV2-NfE!N4>_R9+p%os-Y8@{$hB|Dy+m4oIsG90#6g z3Z_-tk$CwvaHI1S-U_Uvg5S2HS5FSTWe`F1<36HRuQ7h{&7&vtxPsWyz3|(ACfTnN zfwxAjIX+S|S!5$Y6<_+`Y~e2E;j&cv{P%sps_iuA?jIUmB|;3cnO9mfq1&Tpb~kdU@Y7U z0AEg1;+P9 z(Zk0BiR;E%tP`lA7v`=9Q^7jAz)YE6@F$2g35ny){B^Ws-7lWAa}#tmuo&_}1{Upq zgKP8ufT@5mM(`dGnbvN2)2M}$?fyWhXeY>5*Hh2gpSZgccfQSOhP`SrNMQlU$bJE9 zF%4d5jyxLgipTRxoTonb99{AJ8jy%ctTf}YEGFrwC$JCa#0s*d1AB1I-1T_!{ahRr z*blOE*KvRIE^;oA^G^jkz~UWQ^qb)}aD1)tkiWM&>t?p}TE2u1{m8MIB%f}j-_<9ex1c5-`Njw1P>zvhpn+~jrqZJ~`_Si78b%K;h4g?# z+0RR@b$UwumbD)ZzUDq5>H-P{X~fdC9-FdDgMPbIHzt3R{cGXjfHRFkNG&wbKT4N zvm{}EYdTKgXJapE!zPVDG+h=$XRYSGU%r$PmE)LCrgNtQ~; zAH~WouOYpq&SK~IBM9jw; zQI~@~_|K=9`yV|g-W;E&)g(Y>x<;dEsZ|J?2^U_XUp_Z}Bv-R1_sf5CP9DP=0j=V+@lxx6L$UJmT@}pm!lIn!Fc%6;Bc?HEr7HtHqb!b_`2$SCGWb!VvKNKm2q$9}6Fr z!R(Do@ErS@W8|w5mnLP1d#x1rh01akUCBXzDRBS|jmn4~5Jqq-vGitBN(T~lUXh4PSyX22KrTegD;%SVa^89~3h1B!LQ9QMI1D-QlM%-1Gz*1bz zxw*)m*!W2s@Iwd54>>IM9&*z@Tl7)@rRrYH9_&fmPzpN%t zq~c&r+Dfw9@(?*9c8RC)vkn_&+ZnNH&YgI8A};u{9kpcz$S$+%)S1hKDDRSjO$CYg zMoj>g?%$6Alf&qqqbo3fl>;Hym2i+TAS=_j4!O28@A;8h;%L-OwX`pgYc)wstl|~S zHaG=q^V==rbZRg`Qmr!R-N_plfptdovr*7mQ%Yg}%< zr+Fou=a@q0*JoPz=9kg(?nRinW)+s^Dv>a`AsX0Ugqo>wxbSBd(C&+Hw?Pq?2NV(d z<_0uOZAEv*c-1MJn0}iC9J7;xmBY(P`L0M3*rrj{!R?_ptv&_^#KvgFr4uk|vlYaf z^pnQo0^%KK1u?Fn?4M)>s5>eQ{D)NRKFo=imb%vXE`Ue=`LD+G8VjZYbfE1ppMs86y$c!&5wQH?LHBL zH#BjhY$T|)EJNFjIT(Mt2n=pF)1zKtI98qx5e5bD-a7+JAMM0=eLWiT!x`H~7ofxg z3sje?XAav~Vo@ARONk8V*G52>dmTM?#|(Em&V*_i2U_wV0IY0oSiHC}4X>`yA$iLP z^EUk!F&rOjdE^;F6ZJp#(&BCt>AES5HgtQ0vo9V>ZRY=Xzljs}O@b2JeXaA1&zl?1q6WvtVCj zJsIBMN!e{&|63{<1Ws(H*PW%HaXKH&oO`I?kxbafJ4JeZjp@|~`l!iu1mm-I5`(a> zba;y|IU*E+G4ewsIAE0C+53!CcP@ls-yxpBO(A;KA`xO_QS82ie-#@9q+!5ZPni$ZnR)^ToWHi|Mg4|rJ2w7JF z#zHb_bz6$Xz9((W-`*oA@JWuR{l*IpWJG~XXEYR9r$cac7RX)8fGLBCL_1Oh1hg(Q z@!4s-13SW?sof7wWe?z3p)X#pdV{+8=J>bm9-dz13c|3S{1F@le-SPZ#eFu*Hdv#q z*k_U=cas`P>+{|`W$FIh9dztbA|A2jx{WQ771Ax=iKX%!@}%S#U3y_Jyg4}o{=52^ ztR*S%u2uz~Ut>talVogvd5u(mIt=QgYl&*UGwMpD;hNrA$iClA7Aws(;Mto>(!TjV{VFB~(tZy~ z=(i8BQ}+Y8ZYBvjWsP(+E1ftfSb}B6UShB`7#0KsGWo6QSQo}Q#Ewm*DLIBXLqGzv ze9W*@@B-Ztlg0#>9)*<)wCHhxI=H|515}8L!qhFf#C{+fm1EwJ`m{ncP+SP+InG$! z)kg1KdBSw>`9}}=-iHIrxii(li);u zCRSf)u$Us84i?I1(dBh8E-^@>zdr7Ul{ZA-JIAp*kDYMd@ig!C*=TIw?u#ET7vo;e zBazl_Pdy7nQ9tb{-T3Sty}oULnq-?&nKUbyB~;27R* zTYwSX%yp}hvBKaS%5i-EfIS?$ElC*qh4Z;PkPGoS-9<0^+R_f?2Ar{4oVuU5M)bst z;mp4m`0V#Cv>N4M&i=*p2)Ryf{_=*WH9U;c%%kl^v*C)y9C)#tn-j$fdCW--eEg%9 zxL3TQ&v!QAi#@(bx92cUEz7}AbOl@rkR)vOu zdzX*N6E2~j#%2=k&AlZi+^@Q`RgIc7h;a@mt}iy0LH>@1(hkEHq^vIms>6bKDPQMU zq#oP{w;pS<20lfQTOUJzlq|pn^{IIB>|-J^!v%MCSMolNoQ97#w-eE7N09iXL|o!W z>AoGW$)}(MJnnc9b;9oBy#BMOS``jkO=~UoY~Z7A`%&!j7U3_HC?;nD#^K|QGgz4U z6n18GGM4W+PRXo|Sh^*K?9v>E7&|^p;(RGSz4P#OcuR$?~<*b-@tNwEHb z7#}qHFve>e$nCHNhq*5Bvi&o5a$Qi7czOP#OqMj)uS6x1M+Y^$QMJvN-v7;#hgT-@ z-Bh~}sbGF7{mCcK1h z)GK4kC1q&-(G2jHlZ2-VV)Rzu7&?wW#8Y=dA?TDofAmi{wyF=(LZb$9>3$XD7mpIl z#f$0wkY0;O=M&`0I)AA9?F&9n#bHr8YE^ zq7+?toSe?&ah+a)08QTF&jS3_Zwqly>OSMVk>gG~=+VxWI;go3j@PfurBb(4K>6Ta zJT)Z7&Bz^aVz&T$jN3~&!yO#HM%Y^AXp*;Z6>bS;G4Vw`{PW6Z-mfTtyDweftIJmm zVjJ;E_-Ak~xk}E3H}lH4?C{mIL1?>1A8NSW{7O$HaGpOEPrEIowtd0yHK_|WnNEji zJyYSaoen?YgfnKlRl$QtKXKi%+1Obl$Uf|Ipzob`(Ntdn_NAQwd*YubTwEAO*5z1$ zca$LeUqK+oipD_Tf=eK?C!dPnNI|P0DJ+;3N@N%ALzBb{jI7gMGFRd(Jhe&0I&qGv zaw!isT5^8#ewlcq)u_fYVq z9RJni96Y@+k_;zaf%kcm{P2%fto`F$I?uiorpHKv(7smO^m!8hfF%-1-6{OAfYo@* zP8Cz;`(VIM2K63)pw9+A(4+;@tk~mwU|**}X4=`XD)Gyp@PG_^iRBT~)z6`A_Z0Ad z^aI-ZE`xegDG8Ho!@ezJU|6>mjyDVO9WG~qW=RY@UG7R|{9XzLC%LT7u}yd{w1)T0 zND%)Tn1jODS00pkFc%M&0PAc)){kd{-b_WV`yPl-`in8Zc_L1=kY|nao?_19Vq9LZ zmytT-%JFJONRyN@3p=KxFUvwS=S%u^;w4RAB+gDa@`UK-)l;kLEUdb;ouNw#s9<6i zjP04q+MVUHICEk_&P)(dsRaPoy1w8RJ=jWV$ zj;G#aQHPn~xcmDMI2|coOpQtAPZ`o_4rNdKcOk|8RwluxQs0Bbgc8`cF&iCc&fu$WETVRI zRQdJ|UwFD+_c3b@=dgSohg~HHPo6F3GWwtfYg~o^!xGxl77Gx7T5ccDM9hD=usz>eOgD(x1T`umwV9Ndlhtd zX@GCA6(f4B90JUnh{vxa^xszn6UP?vKaQjkEq4e0g0nw(>fxth-sBqG-sVY`mmofD z7ocyWtXQFQ#he>u6FM6IhN8_kaLej0((LmdWF$_(^tp2|RaBq7I-msTS`E+>oCnMOF0c?+D93DnFlqC@k465)tE=owk{v6X_&`R&VP=-rWzS5GuByug#5Z>GxME%M{2>h=EgSS3IUk3@y7u8|? zw0St^R|GlO#6YH+G5_S}Zkk)i?R&Nafzp;mSUCSWv*1Q24C#)7aK$2gw4QqhYQBqZ z+9%NdRj0+`ImNJRZz}TdxWNaLX{1D95`R(CZ78AZfYnTZh~>o`3wZ~$eG+8P8g-Hv zmxS2u0grIbiEdowHWR0uRRpc?cVPI8JhuFEBI;f1_>aVHS~w*R!d)*17Oif8qVH3{ zQ@|EZ?L(0*ez@_PKL{%X(IxYRaN3V|B)V@ukxKP|Rny*s@!?A#Dx-@ZB>iy9P%3tm zN7LE!Rrqy_j=`=&aafj=K>BYcL6Au|%y7(x=>p#fyTXj8e)B$Bed{AmyQY#B{vJC0 zngxlSX^fZFg@fy&c+O*Ug>FeX4Q_5i+#A{$9v=S=4M8^;kwR-$NXs0BTtxWo-IDNW zVj{9-aqwGsh^b~@Lg4Rbkg$*Q7Whp^^~C`w>Hmev6|us#aRO|k+cvsPu$Sv^O=Lgt zk3igi)7k#+5zKTfU1W*m zRQ85V2QRLpojA*86Z`MCNdNtFcrsR!NPdgN_=QKoKzjmzvTHKjQB{GBQj6&n=P~kl zXd-LPZzMBR>?&3_oo zFDekDvLY}uoey5t(G2e*@@)S8fHXxtY%ghsl-{!>agiRC;mLVJZ(+hYxR)3 zfv2g0&N)n(B+9+rF=1@#i$%wG9C7i@;@NMiOlWEhpv+{+30Y;QP3&b206 z23|o&l-vJvro=O;fVcm~UJR=8<%*1nAU(I4tk;;#YPFkF6W80+azZoAmCuIY@oKo? zK979ZKS-JiF5`4 zj=z`>n?wYld$uwRIsCj|=|L-N9bJgWxI9rypg-JrdyU&eoPqmsQ?TUDJ~VRKLWhqn#DY0Lp_XHim#Y3Q3pduzr*6xUrcD;O?ch^hm4v=F&`9Mi1#sjH2K4T^v*ycxImCv#mr=zeR8m6 zeg~}D8cVOb1;L_eCVc;B?wxCwDW+S9!mB6IAoZY>7AZW&YtKIN7FKFN=-UPCe`#6F z35PeJ@9+eCG!*&$-2%AyTLaFWI)qiFP0aGsE%@PK5>^gyOfc;iu$5!)EZb`ercV(! zN3BD>+xx*oupWAjhr;A_+W6YD0AmNLiKPVB8SxCkBSYN&%uo?h?|nhfGm`vioXflH zzAq@v8YUI@u8^fIeUjEh_ep#0nt{RGqri?P1Mmxc5Y9#0t++;SH%Xt-izqb=BNIL0c)1g7y0}p&Th2gp(%(djssuD$O zR$J&3IW3j|0^A&|y=FhY888Jw+hx?_>O_ufA%e5-&!?g}!MJ#@Ejwrb1bVBHbKsqj zBkS);ky?QkH2Ee+^)%+f@gO~#kd}q#CX_>k$^%|cwkZZh$>U#+XEB$G@x6WQ@c6>V z;L*{BUtDfkxRzd{-`77U|8m0N_QG83PvkgjAwrDbr+-Yb_j*vt6X&z*K4IFv+f0wY zAQ5#g=RMol1E01BW9I=Etm58lU!C6yV)o7StJWMaaXHUx{qBu9FE+!*mOeQ0wgpC+ zr#w$CXBD(GlsfI!;?EE1ATQ2)BPmHk|Fw6BNXJm0iV)B!1*Byk%~)T_ec^q!YT*H^ z({O&87fM+7W82Sunp*46>sTs{N)L+2p4Kl=<@g-<&c|M=Kjo zEOTh2r^OZ7`0>q5SnMlE(9I(Dd)}asn-VPU-^^@K%fpV&Q1nWf$yzR%0NXwQJ7%5* z)~bhSu80a8tvp9BcnssLWmRz9%$azcErjM9TS&qrQM&VjKGiyTmp9|QEW2Xwe$+eM z1-!PSDE4JGe`2>98uwRG3*`)GXwt?R9NK-A7vKojQUxf4j*c zG(PnQ(=+aop1xvm8&IO97cwzzLp#oM{Y~y$oB_q;1lU;LO?Dfn^Oo&?$#@y(6aMc` zcper+KYWtnyE>nuum1C8bhmzmZ_>IXcPU~sL_xu-+mKPW2_BXT!;TL($(EWy^0ZPJ z!n>59VVVLa&gJ~TCmiS=K5|~PFj$|zlxR1dp@R=@KtQfKPIE{ki{uGoF%#k6 z8(aw)>eX;YY8h;UZSfXyF;9EGCB2SAtwKL4yt;O!DoUt``?#T(D(AeaF-~w z4|$EdOJ9){M{VG)q5=6X(?kPP<)*uF*#Q0zm;BVBZC&0L80*M$RzuX6cib&U4S;XJ3JkoixNICPlcEz2ba4M)m;dc~;Cn1aoL+GK2VEi~Q9X2cmQQfycaohdKS%rpc`xp&+} zZD}~|wj(usGRV|w&E*&inz(yKBASfb;>B)E;-7dn`Yc)XQ zk21K*?Za2|b4ZU<6t3R7FOfkYc=Gv z%N0~_DaW+(9&D=I4c|u8>0jc2$CFescdrzTJfdLNeVI37=XPGDhiIex4cU`6&E zK+C?_>>YO_I{vwr_u$?I*b*{ESQ zS7=RWfhF^mx%bO9Fjss-PFqQ!QQ%=%?cL7w+vbwfCs#nh0OyMO9FI;DHp0c_GpJBs zGzt_MHH9dpr*zlbRd}#* zIT++5;}q^sc|G_HZ}|Ow%oG!%8|9KH{ZbAeCvAfezel9t#we(FyrPD_M!+%bDKi|4 z&pnJW)!_&YekqG}Qj+|T&$sb$mlfWZ3t~(zaUG2dy)?q%)?sSNo`id-h^A0|4JPdOrm!q#~0M_}eg544fzWtkwl{*T|o+c#F z#@=J_4&IQLj4EIj$4*?-2?`@MRZuXW4cirnpO(6zh?+R(X}922EEfQ~-~VxWu`ghH zPaof_<|7DmGmi9E&aK1w+LmggQF16zSSdvh=9Z$zEpu#p&G9=UryOC+6~kPVJ=D#_~S-dHDg103$&rFWONlE*>L zsC+g6&BEWItW7J17mBf^d#fSj%T!jQHI+Ua)*#gvSuzx#2}kE1AmY0^(dlY3YU`~b z*IGr{s}6gq^xai-!Yf^N+4}_8yeOMaC{u+~5K89F(4ialZG{u63AjX7f|VRwhP$h? zNYkO$BrIkE|D~@Of0c|FxGeuejUI~d@91jr{^(@lw)TzmyXJRX(C`$xD(}PGc+S_A z{|fhe{v`i-|FoFFPsBAgneZQ%)1;$Dkf9>PY7ieP{n;3%<)1}$rF58iQ<5+3@f!GZ z3~|nGjxTro3@V(7hR-}HFq{4zCQhEiwg>&-Z9MY;zAdg|UQKnyyJ4a5Fx?OC{apuV z#3Ly8i<0SQ9>Um$8cfQ{;r*?CLDzCH4UMZ#nEs2>TQ3mo8GGJ4(>#useUq+?QsLV< z=vKHb+k{I`yuwt@VHWUL7v8ZKFfu(AH(hU|$$EoWrSq8a%Q5FX+vng}k`9rakPc(1 zhH$$}mv6PTh4DDB9(&RsVne4W`(oD^E|{E0(hhXt-O+pq|BwhfKC6O7^K(W!K8eJ- zWuW=!DDE2AOD79FqB3LcXv+2OYeuhQ=($u>;d1i>nlfGP;Pn{lj~$nHqI`l>81@ll@C^Et*(SxO7Y0<@r4Cp z%VFosN&L7UHTZCc0N-qW7}hs8Li9ibxPPbtXFEI0uvDOZ5#Py{!fg6hLkV^&<)Yn! zix_XX9-g1z;}nyB^xa`4ehu}-!s&+O=hIxgFWirwE^=(`O}2{U9f3#A!SvPd3%Grr z5@y_y<&&A7c+|-go-0tCZ1V*EaarII;X&krK`0uM02=bvu-;V{=ZQ?h6;mCcKKKUu zTF;^VX}9rSPXSp>+FoO6$|&sFMoE_c)je%PeltYft>}q}*$IR5J_@ z*Il5Kb~&TiGHc@4K8Mvh8HrzFF7g^m3UT$Ft;DQ41^;|^qe+$Su+lIZKFlYu&Gik6 zg$eMTWygrll44r6_%J+)uK~Orh07LSA#1;UWBRmLW17k*{5B_#c$TJs!Z#6UYhFZS zelDi7_lQ7B_EBQyJ&iS(F+_Hn7(#^lJL-{h4Xh3}^3J&Ou!4TWO`LyN;t7ID#B^A( zn2%AXDuG#V0^8~yRe4G5MCXa)%B_iTLl4x=`DlDKgg!m) z4O2a3AW!-!Rm>>I3$hBR!g~Y9lLA3A_5{Zact~4qU!!WDOV#X``gm{kS<8BTP-Fq8sm~V+{vsYkQERwZ?dRTeX1?X>E`rLE@gx_<4;s941w5TSx zB6zso?FVo8jU{9~t3&rz5zwN2sFpND{xh}wA4O;45aZW|;UsAzkxE%osiaM|>OIe> zELlP&X+w(=McJZ+N~KMueP6T_Qc0Tgj3}ujNwS3yg%I)+!gs#^plN2_Ip=xq`??rs zMH=3&ii3zaO(J%fpkc*NoH03$ruT^9z?vP9<%x9SuibE=Y8IKKQFP1~~> zA7i8pk6Q01Im`O!NZvVa=VEcb&mnm(P4*~+Ha2s|)+9nSD$?0fpDP?Q+w zKQflw&7%tp3aI$+3ZnaI4^F=_Q*bl68AD55;b7oqy5bGnnP&gsGAt(Hsol%C^(#*U z|4=E_S@jTQ97~8!Y%t#LK1Zo~3^d7(!Ap&+V0>H@xtx^@>-Af)ctZ;=Xj7ox+Kk`s zlEm+PpAIYM#DnBA#Dc&xnEd1~EN`E|H;&rGed=l@pPe>fS#~ERsiS_ zyx?KV1*o{)&2J4sQhPfTo~p5&h=ex2yBz=y$5X&2{5Co8FbEFrX3Su$p+O>|oVMat z7~Rli6xS08mo+xf+DFf5@4O>$>v$Y4b#cYxn>Rs7s4Y$qVSXx+bvX69EqXRJ5MvPw zoIl$PgTBes$t;!wMRsE$^m5aP#WimTS5=40wB4HbvS6)%m9P8~qoJxchv*bW88-sG-!2nIWy!CK*U=-{YNeqN}mH5@X4IXRZJ znmdhq6!+6=*m9htA)xK%5o_ipNrzU zH6d6(Ji;lb$@7Ig7GDj@QRE`r$I!3+V(92&g_Dmxq1&2DP}w~nJTs%{Bl*o#sxAyJ zSbwClfzz2UvyJv9Pvd_*u@5awSK*{UE9k8(;{rm9&`HT19K_S%Q*jUqwVdaZxTPRc zO5j!I2Hca(+@B9mfU?+XPHGMFuV!sUySqQh^XPmiwf2Y3%R5n|Qx028S?_qiHEvln z7gv>!BSrCcn3f|%f}D#v{(lK*!e%In@?+t`(myoqxgRdPXa?P1!XWL<0xHl?$D!bp z_~3F0&Oc#-L4U*<8<5RKtit$8C8-#nAx8%%<G&lkwv@Rh-=Ah&o;?P!sYo zs5*y8HO11c-SO}@H;!?oYd8UyiIXbX>zSuRdyS6~VJ-ul*I4o$S(m~0fddv#F2FS( zHNd|t6-BS;qvg*q>bvqfci&N+ zQ$hA^7!?X}g~E}Oq*hfNgtuhFbjH*S7>1PnLKR{b%wrs%4m1r1_u6FlX|@x zZn=&xta`ehbUbySqkDFs$=;)|GDa2({;8me>ONxETTS%s#-UbwGCoXY*~qtBae`w! znjA6W8>)KJyar4Dh}3wZY~&8!|4z_>U^mz#%y^Zn_ksO1SB%J8MY`|r#>Yz*qS)GO zh?w?@7M6_x-+^!t3;RxLSvESYat^UOX^)|g4I!dNl|Guj1&+siW42id>U*EyYs>v5 zhL)))ylorK`Mi|zkT;V0+SL$WXithoZqY{7Tx@@y2@$i)XrAs{a_5j1ss~?E4Q%u;gL_Y(I5=& zpN_+%E#b6UGZ~$PL(#-mn{QXKj3mS)fqv*_u++FuEUUDMwq-gruXcb?vo*-OC}TXZ;xi??5c1{ryVKLK4AKutUXVB2lms%`0g`k4z zd^c4!G|9ciAFNk^kv5j?YcIr`s#a*pFXxNe{73(8PlO6@7qqJhq(Wmu>m(*-pmN4{ zvSSx>xv$ZM*y9E8ic5rm*r!y;_!`$dTc1pdHv`|wRZyz7jyBIa0YW-!IH|GgaV5*V zwznH%QpjoIz;@JzembP$>p3z|wTmb}yG@M!BcL`Qod~sA?UOOHZ zhwFe)MTa37Kb7&B{9yGm8Kf1Z81WzwM5=?Je%5rZ)J#D2n;jvjig6p4nZby~VJJJr z-n)^Vz`%EsdvnUE$z)+lBFQ*(~Zu zEOpA%N0XcNoJq7iUu2O1WVx?|y{+l!YjTMyGd`_vkQi;8dx9vxTS}FWE+(~t6Ogqs zoy4SX1TrBPCmv_qxP}2@C-j91JrU5MjqJRq7Qr|KmMH6(fkD4Z>xKf?(!Qh9QT?4S zURsxgQ+gJ${$vnzexC(FyRX;z^0lawsv&BLtDqen3(NbD09{={^Hhs*`5zk)anAso zBWZB!c>>x=3)63nY1lh5mp|}+CD_%wLWc-@K5aCBG|M|=1qouD>+7W6(*;WTE;QWs zAf|yG>$sZ}{q!R+txFqCUOgkmHb(d>BNl7VnS#THZ%og~ zf5QT9?>8kp^i2ysZ;FAupb}!mIRC@*#$dg!9VgG`Ct2CuM6YfKR`1Lur4Q$V;Djgc zKCTXvUHx$Fw_Q|BrHvTgE+oC>d#JV463m;&SbQ6P5O>~M#=*S7eV--<`h9D_DmxTS zMaJOBE?4j!x{afMl1RtbBHZ~bl!nwuLCpSH&~nxR2Y(zSq4(6FM}HgmF8azfY&{77 z#wd{LZLi3bu`i5!PXeBaJV`ahHesLJZg7Y>fsZEKrdPkkphepUGTrN3ocUy1cmk#&a)gJ6D5zj4?!L(Vlv3*Pc|VPn=Ml-V3h z+mCkALYZth|GO12HHtY6vT)oQIW#nj#i5l^oN-Vz)@0;i#nbI1YS{vQJUIy)?iCXQ z*R?EnvIc9dZ?@u!}e2R^b|?QPXgv;aQa6qL}IaKe*={=)*!q$_FQlBJ=&Ts zL3z422|mcht--h8#V$(_8Pg2ggTIjCtNHZOzi(8yIfKp6_MxIyI^OqjCucRT;g4aK zjOBQ!_4x@b{x%VusyVWQKcDE=E8(oOlF%#oM=l>?4Eep^aG&uP=ufX9|9yGPB?oOK zUWK#Ck=ainD6tI3zMDzTk3S8~ENA4X|BayQY?5*$g8$IuEZH#r16Pu)1fkC!;9HGE z>M(C7XemvjTN3Yq%4H?0Fo2o;x)z`2%- z_;rOTY{-_zOTEA8zuC{o#r12MJ8ujAd3Ow~d^SV+#2;uDtcxF_b%}Su7(AbK9Ur~Q z2Q`NfZtK-Uz(q!3;=T3c@r_KF$6^a9*Yt7S=RP#l&*2hR?}uLtzR`s$b8+u~^5`++ z0f~H@$vyg6Oj2$=B&OcVINv}V%@@0Xf5|hD>3%hs%KncxU2pKmFs{tNs0yebx<#i*$~jnZBJ^$A>E&V0G&4X+L@ z7{POgrO_~Q9Gsp~3DeYj$YCvS2+Vbd%?tMsyX>{77BtK)dM%Dmgf(fu=63qDG8{vV z1F_03luK%BM>RX+nEA6eg+!2n(3Q-CoN=B&T9>*GU^c9(YKMC3@+oSRwg5VJi$>J`Q1m?Nsp^IkKZX$2^WNaNnM~OK4+~oFVT*P(w0klJ zqs%B4uU`ybx&p}EK`%O3;J`kkI?VmEANp1f!<2{FP4fXst1h_=lcAf%^)ui(p-1O9Ps2D=*Nm&!P1ttowdQhNl>G z281WQhKD;vaM=1js4B>Sj)otxY3IRHt*LzO>o01#RuK+&MRLto9AV;7M+i}3na#pe z*nIX4Db1A-l-vmh(|A9WSkC-N4eFp~*hHpQ72(6u2KxBWX7DPWgG2mV5X$Cw=NnDf znYIcHZ;JC6s|m!tYk(|jp$F$rh3Lcv@-FZ&z3Vp#^H{cGYIu* z@|5Q{LEvRAPM?n} zqM+%<6v4u+2cT@}0K|$er}kqPqTzM|8TSb+683=DKa9WW#C~I}q`>LdeUj8rNv0Kt zk^V%cw&d5K!havBTRy-oKV8yqm<4B*Bn8*250He-eDa%{1-ope1obCRV94Fiu#3-# zpEq^UXU0D`b={Nhm3>A+~A30#C%5&LQGVDiO|b_$jX zPUTDup#P2#{h^%9&nxrARHhrzK~jMQ_1z~>$#`gNBM?E9zB zak87y?PLKSUzLU`WsC8U6o==|wBZA@$#}vhANG7=|6~29^tGHa&=6U8*4KiS6>|g^ z_#S)@ccG?)oU$$@9*wW*aAhBxjqiiy z4jO`Y<6dE_<{UxSv0jiER^ct3REMfzdPL#mVsIVs1kVvMp6jrOV3&OqV{z|*?1Fm` zC{aN#mYl>NZ|aQyo5XsEKi^&5elUZ%MeQK#Ks(U$x6qBJ3Lm1!g1X~xPR`z3koJBB z-Ss0KvV=0gs*SmLTc$u=QXoGvVu&0WwI*-X8t9bBWSkQ50PlQ}11Z%WD3p7Sag}~V z4BWx3AeQ}w`q*rF9AWwB6b-Jz0)2$MPxg!H$ zF+LM2l8qs*Iu)CzkLR79z60Exim*dZlb7(OkmNK;@UBK_;gn0|w6@V1swZ85jItbX z(rkj1nZ{IjSu<*SNTKWaxfmO;5#M?o#)myosQR685@s!;QKo$MOkRbb--+{7PtPae zGkiFmwYPBP$}zlK|B6Tq&jVYWU*O_>2cba53;PFlL4)KNAo|Z4_bZF=f*RPpSOO=^ z8KmPMmchtWO+nt`NfY&X|+<{5^q`+ndYZTxAc#4a}QqGJth~&d^f1kDts~ zDUFA(!ZI0oBAen3KPP;r(J7KxBo+gfYdR@!n-=l;T@Ct=^&p~NPO#YSBRR>k)Z_M~ z!TMPnFkyBvEPmX>Jksg--_NJqkZ~iYmM9^3@1r1SyQl_vZZV**@gEJSj3z(+dBAh$ z`Me`F%OJ(~GJ3o+6s-FC2#TMZgXzj63Y%OE~&qY>Tm6U&QWFJv>eQqF5Fkk6BZ^G|&6p}P@T^F9$@8CAfGJMTc|UKo{34<%sR4H3l~ zz{JpiL)ZBNt2MFw3S$$z@X7=Zj@-qLI#>R|7cc3I!~wkC#db*49pVOZ;kfu$7@`5pPVRs2=m{DfRdvHud_Ij(t*3U;KviL z|K3slr>@m_L3kJ(-hCqauk;v`_Z}FolYj=;L$}N8K>DU+xJJ*yp$kj+8;blOAjFt% z4P7G$)?(k=pVz_1qJ*v(ETw;2rFk262=NykItJzmC!i|)A_VS|6=;37#&@TxN#oqX zx{e6uhb`-a)on67wTdB-`5Xk{x=pY$P>9!G{~Ft`X3~eFWvH5*hqu&zpqIi1yghCK z-ao_~f@;TMRbeiCEqo6vdkyig+)MJwuAH`w&fwuFLmxPe7qrK25KK9Gl&0$WFeaZg zc)4ovcTE2bvGbOK@-<=DGnj!D{vD8c03QH-b9z%fHvC%iZ5lbU@glw?_B|V&t1gLdxTJNtu{$Jb`BFqyguk| z#VEPSyq@*aIOV)OX7%sDPcmm<2=#mk?V7qY{<0*-tyK`HD7?UgH|t<; z!Af54+;kLvT7;wI8%Vv{2PPUPIUfl(LA3Bw9C2Jt_D=|cKS6F-8UBaXzOW`Xn?g~3 z>V61VoeFt+axE>C~s7o7cN0#0R}$Ml^@%&CeXLb`>V&9{L;2^rp|u@b!AS`FSr z(L{Roh6gX^`*9kud5~1c-y!GEFGjuNPiXwDg**$57Fzo51g>|gg@qMYkj9_Jmrw@I zA7y!(-@e1t$Zxd3J&HQ(Y#^04d-19LS$@ZdI}oaKjR?P*0HNlr6DRG@`1aSR`V(`W zB|G0ah^q?5*``Bye1i*U zYPIlnUpfiw-%Sx55z1manrJi-ZUUR2d;F6|&*6nO>n@&(B$JNxsr`$!53y9vJm{YOBkGkgjlzbg5B?<7s7F?Ua;yn<<(0zqmDCkR~=&P-y{Qgds|juEz2^uMX9h{yBU@@t@WEF2IQD0xW=?EN!H%pnoFUyuuG3ug&Lg9;$l3ohBH@P1zo#sItTSPEi-58n6Tg+eYj=IS~6-|z!iF^xT+Qs(k5Y@b7##aw8H z>}e3*AB*vd;{qIev3_-lo&iP#c#=>>p%MZuMSc=frow-Ms)c zcijY6ST5MDe?F}U$p_=gyTo5!i`Qww=c~MV0^^mM$w}*4I%N{$?ru7Y4mRFY-B6lm zEcAg6hlJsQUy|_L>@jXT5{ps`i^!KOdkkW31w*@Dnw8YXSxLHpdyKkZs(l~{^(umMd*b1E&VE#wYA9%n+9+7VvOX`@tVghqpsRb&!USPsXgDm+ z3;6R4EW&4z2`=rxJ10(YNg?CJ-$qXN66Ni^3J;zv<2kRsPozI)k>jto;h`P>L4sB` z46AKIwdqPA{-zN&#%zRh{OhQDOBy2ijdZ;`@ zZqiGzS?CJhT1Eo9vtdS8>Q$ipemQ7miNWU+^LgcSFVYBABi@;L&mr-P8sDWZkoB~8 z!~AoB;B=t`p1Fw#j_F>e()=bX<5pkO%fk|Z|bBH&tBr$oD{s_odxdCkHXv=b0JAoi&wBx3B;VTVDR31^7hL; zRL@p`C89&r>*`5tw3)^mf7FPq_w#|d_X%&}EDOOg1)Of->i^f|i5| z)Rs}gprmVP>h+WeHjLxlJwJqHF}mpI_zs#JB+>t>IG&r+4Ao(>yoPrYf&%XsL_69D zns4@l-31|9`K=e$o){wh(nc73xBy%X%F02w4ET0&|{q5Dq zPibRu)Wn}yoDvbtyg!L2y7wQv$ejp%elO^sv|qTa4Zmy)nfY8-lnRx=jX2u43Cp-+QYM(m0O z$R@HLcz`qP2oU9UnZ%>l?EyM_R223rNs|&49*&#iiJKo>zzO9dxXD$SMg&B`=ww-* zyGs;YQEY%Z+cR|U4qd!a{~Q*a$z%?oW?Z1Y5`EYm`JKpQFlb$bk@ucJhOr{a`BMw) z9~9CzGaJ!oN)C36ucs>WJNe~(8obwU9+Q9H4v@Lq^Wprw0wgnk(FhB^k!!RyeEhi| z=INazE2liT>RFY9VJ*YxGct!R&Rocgk^6ymOAYC!1=8petxdkh_R_@JzECN989bU! z8NHbJ37k%kfZ?-V=y{xhLr*@#JcZjZ+ z@gB@45hcZt|8JDGYXs1^-ycx@8!|V&DldKFN*HX*1CuvL@%b2E*6ET!;n%VP2cgTj z=&=S@`F=C@=jDN3bS;79j!+$30rgrHbsjGz1R3GhaGibU7u_|1OVN2G>#(R`=&X!!tk zMPEUFnKDF93aK*?)`5B7+lj{GY;af+3DS#h@nem~3f_LRM#%(S_*eV_mMmKj2Yf5x zlSQ&|!6Ic|X7~qOfAKs1lrJMfSF2D~Y$_g@_K*V?#`I7*fOmR>p{_Xrt38?f_*)9R z}D@#~~YcL#a$u^7zGCt#2Nd7SIh2{Ss}AaYcQOr3Nd!i}?`W#e(|mbe7x(#3h( zMbtPy9eKg-DfXC6&aXf> z8v)+koI*b-Qt18VhC=cmsoBb%+>aHayjdy}VQic{F4@Rn3JUG~(A)wL(iS3RSpn!W zo3W$rO%w&mv9%|g6H)Y3A?+KN-p~)C(M;++-WlP}t z0s(}v`CUY}ELwBKHG8FVLPsGi8can~C7fBkswjOf13Ne>-IzD1 zm%=(uhU(z0c?c&jW%;tOFp%x&rWPOCK!!hwXFuBmvUgl2RSO?+XKw!>Z;l)$eK*I$ zxoAl!ogU9k6RYRneXIiy*OzkU{6yUKRS6QxWg+TuC|zk@OO;%=(4Vn7n9cSuD}GMG z@Rz!9os81S^CNL^j5D17I}t8rJO_`5XR&jt2%P^cL316y5!<8t$mF3+c&wo*SR|B- zEsO`}r!9$+`R17TdnUYAoPb#`${Bl_kVit9IF+%COotfUr~L>Tm~Ml)r}fb6{_=EPXcB0f#Dw$Rb`2 z{Hnh|@9N%$k#9lpy+aNDbxR4{sa;V!B?(C3;^!o~QeBx!I)~!hbr2?QqFbHF7!!TJGFe2py zRH{V4p~rhMEiInhaMwX4P7g!XqhK&L3eiT3?6sXpEWJ0wB=I^3^%a7+--@)(@;R-4 zwvUW)s>a<1*#1nY608*tvMkpmoO}Kx4OnyoW4$*cKQR{EZk1zMftsL8xkW^+;zM^>@(<0ar8UPIxGaK6>9aQHekjYMyqk48Uh`G2PO^P3`z_(>5P z;OV;!*jlcFsy5fj$%*S}@w$gNs+kI>co|eURTtDEnajr@5_1gal9$J7abkTwG^xqM zf~N&w`7?pGa&thTHI6o3isy^(n}ens8@QTv<`}v*o&K4Sj+T#eK;q^W-1_|lY&q6W zN__Uw99||U&X|cGl}>?*-&LA7*vao3V9sts7g8a)kC=zE97oVNY)HIA%w_dZ`~3pQ z`FjR?RpxVF%3@IE{T%YZn_#H(JaCV*fLI%$? zltbM$bLMng4(E$k-=Li%_Ap{sPkrj`;a{aDs(o++%}F;%*vAtr zf4v9Xo~LmplRngaI(!rpR{6vJ;i)L`WHRI~T*{XzHYM$Ai$G0>xz_l>RN{v(4sMK~ z8@x?0EjS#dw%w=7%Ln=2-{<3*iOHzW?#dG16S<&Ag)mA0(EPEuYZ{$2i8;m!mqlZ0#-7+T7aOpU#xu*Aow=U#ZcoAB)Iqi!HD@xe8*{{YkM& zDTPuGsw7!MSDfr57he{jw%S`d;cy=z?;UZ-vyvQMzK3?Pj{L%n^U=Dri?qGT;k%v; z!~Vp7oZ+NHj71@ddDF&`vp)*xl_?^iCDz6@TgSjc0SEsce&-M5&j7;-?96|(n_Cp` z53)8j+}ZvVXn2%N_H{*}%+J|a5%7uJZe{b`j%euM+u@AZ2;BEZ9L9b;K@a(P<6OQm zv0wM)8|u%XQag~V_1Lytv}fc6}ypPNcz?A+mIF3Wuyb{U%3 z>X6ZoLD>Jm2I=mFaQlWFbHR*~i~hRkaWt4^EZ0J4`bk=G>>FpY$&hb&xquAWB|?2< znQ@L)O7%?$2|?LYdTbu5(bC;_uwYT!KjnoAih z!JvuP>sDI*pl4R?#m1%k;8Cv@(JN*-M8i~~;B=3?t;+=GNn7}nJy>q^Loj@*U@XQ% zJ+$_C8jRQ~5}`JBc6+=B-^ZrF8U7Hh{sZV)8HEqt2;Jj&Lk41|B^;Ko6{aNi@e50QL64#Y0gbCZa;jH!sAX z*mY#O+9_J<#qJpzGqEGSmRxtU#@fheIvl$k`p!KflOG>I=M5?Rl(kXJ&JZLYDd@_SJ=Ia$&WyY0vr$UED5a)b42F>cmz{9yGv5#*HmZ%JU!TTVmbSDn< z?PR;a?L@451KED$8mW(8L)V|Q!Scavp!O;Wd%zZY$Aok1Yy9!@Y6~omEyclG`B(9-P``bt%E7LG!&u;T`8^)CXsF&Pka*Oi|XI2PrG4bei=5i3@Oa4A=q z+daVxm1PZS&RE8<8cGC{&AFh}Tf}BT31r|3%l@(FW0&Yy2oz$o;J0Vk+2$|T+vy6* zvqGrKpG+KX8z8;>b~-XW9fzxnXuauT?kAgB^!Az4uJs)xeyJ_7l~{oFw;qsJipNkt z{WVv=(w7sJEJ6#TY_z*POdk0kB{9ydGcdEuc>ALS$l4YOfl9Jqvp$?Cnoj|ZC1&`@ zRE#!_+lN73DV(LMEXcf?#X4lCasU0(m^NSlG4JPNjqU<0j~*qO0S7UxNeg==lfeH} zDvo@c0OfAG*v=;eoO@WFdphenXs4vV);i3u>w2OMO?!b{4yoDGvtAOsh zU?ALDSY9m;Vxa|?axVlLZIYM>m?6s+Lha+DT->mz5ljz0#NqLc+=ATof z@{=a4A07w0O5(w(t&=#L8_=k1#??(c0`VIY;U9BnJHrfaR-1wFR z13ML{h#sH363PKzQ$4!Zn$4sYDydOIHa@BUOw%rCQ5%~G64f>yf)-3OUSX$=vey}x z-8mFA-94c^A)cD7iL5K_OQt5B#_(G@1bMUDX{ogXEWGQ6VaJ3}U*{5E|35t%re)*f zs05VT6be1oq0n?q63S;Vekjd@!&pf3&qU%=qglxLE=OK^2^P-r1aGf3SbotFR(?~4 zylFC|KJ6D*d5p2AG;i@W`A)FBBmqy~_&2Zo53^vyvgFsMD1`hF1uD=NuNgH7BauJjpVtLSr z@nBYQ3WnZGQ`h~;jD=E6BvMAGZ%#F7zUD}-4`^e!%warzIvvm6OoJSyU>xOo$d=MI z*f(7YcKMG-6A^1{mO4+$r@g0dqjN#Ih7l@%{320hCve4vFkEXSgN?EaK%w~r``a|Q z(XI}v91spd4Ld>8*PoPMT!PL@ANYv@L3lw%9lC@s5WhNby@=~EfZ}N>rHguE1rM_r0auRChvonyU zm~qhcEaOQxlK9n1=V(aoH0(9E1GZg)81+;tjT%E@)-zyInk~d(;ktfEj7WUN7 z{4L#aR~-vKWx~MMZJ;uDGxUw7g2yv=%n1sIo&^_aPgp9vvpNYuQ-d)}rXISZ?~>aG zXX6z8>->RAH}pBSiWVK?sEG7G;!|{oel20Ul*O$icwP*Bal)1s{o4f3nl`}c+j`{S zIxp&*c7=Kj+29|>1^KGX&LKv7!Qa6iXSO{eQBxvNmCYu6zFfxehvRT~jSM?GXB(~6 zpTw{3P{df#HCWSk2pa2-<8#G3q`+wcc>dW6t>4n2%VjN^%>P{XX1y#p+>-zc(N|=~ zedb+a-13;52~gZ-OBx0i;J5PS(61B<9}kYD5*mis(x^@=E}!6HR;+;tA2Bo=ooM7% zng%AlXULP?dU#_GV-y8NpoZ0$|ggq>V16M`~2{7W1*Yksx*`4%9>0(q4H{$$!#CSDUX0Yc) z6y4o@uI?v}#U6`lkTZQno{FBq6J0mSj?~roD)kup)CJ37N>CGOT)T{m8KZiNK@UcsJx$x& zRk=qibcx-k`^H_F#$=3m3obP&p?Us4P&l)POWBaep5f7`ZJkLjxlKTm44@@aFhgoQZ~XWlSW)3c_c;cWdkMv4kE;td$cYHv-QCfv?j5RrOB}bX zUWpkRX>dca2z!zWaV?u)Rpifw2_h|+w=R--d)5K3^Ej@n{>pj3>cGMqlVP=1GDx~e zK>6=eVBt^(I(qfSf;cnq8@>c#N6+ARk6;jye@|+El~C193o-lM4Pwim43S?|8AJCL zC!u(?(*2nPC%#}EOp&Q?7Sc+z)-X()t?`X*4OfXVOC;xiSAtX*bd)X>7U~2Ajqxvd;cG@NQ@X#|*~!vP#2e`y0q~mq;2p z1@Q0~1t|LD52}l5`5V?V?g1|fcf_3r{oP}6cgHHQSjze)GYa`Tg#K|kg2ixlk^+QU z<)FUd0GN&Ggx+LRNY-YakkoQoIUA@+a~!}A1h+%;$rI%{#9{pw9QW@Sj_$jRDSHp& z7RT$P$Uh1r>ejUEDsnY z^@zrrIiRh@VSInD0&aPT@VXVlNLtQr(jqH|n>tov_^yd~aNAn&Pp#lCh8TgmLN+*+ z90Wbj2f%y(j6Az*g{F^V*?A}oy{{A_f0Z-@uTaNVVmGkuqzK-b#QYss8QWZ-4$1*@ z=orN~da^eaIfVnz@@WRK|ICAdk?YWv%!k!EcZo<92c}kov~bo_dj87}x|(+#cKDTI z<%!SCAGnvue`a~oP0Q)`kTB+Lw1zPyT_pPGQtr6+IS}N(rLWJjE}Bn!U2l9jv20yS zP0wCHi)5klw zGsm*|{%&G0?}t3k!BvvN;!?iRAV)?8z3|rU5d0|afS#WlnS=2%s5NBJb(Un0tE%#5$a&B|DYSG?#}9m3G0F_I8|P zwit(bGr>sp7Il5I9%rk+CwWJok@l&dVB@eR%LtER*|Jmc_J##scTL3S-ZS`%t5s0@ zw<%m0*A8}Gm*8|$5#i4+L&@IB#J;GB>=QaoWe@p-`D7k!{G4iRw6g{rTMTe-`5ux! zco<$(U&O7xUHlPYH@MAw5A8eFpwp&{eBbVqu;-Y}<;y^AnUY-a&CvNIMQDLcXYphi4uItSs@ zb95coM^=JMFTjDRDyS^>n)IY4L62%S46SMgi;yV(#A$_S z7;tN0iAe(dSY9(70X27_ny<|%n%ZCWv z#7o_?!DxRWi8k5CshLvB{nf&EV2_#Cmg0>KtY_P*M;5!8lbR!8pv-s?*DO!)3;k6< z>_rr0ip=LGywN3M@o_Ntx1Q`i6NZ&%MREAJ0(nud!~78WV5KVxf23n@?49}Cva83z zVT>#u`7jqo{+$Fv%^xgNtXgOGZ5DONGX>47Y^L%w8{Iyo82^5hjhYj6&?NQ+guZ30 z*_~^E{;MR*&2ON3r8{mpIY2}WJn3cCN(emYiK9<-!APDyCqm1LT{zf30atJ2!7K5pup)%bY)J;@t7aPK2eEmwR2OF=a{?@UK2aC0 zi0vV3(dhChdX02a8-rdFb+3`Wj^0Dg?@8m%T7;AA0a+OBzk}VYlkvgaTr$DZ2-hWK z;IgkB9bMPc{dPUqecjjn+|Td*eBM9$oc3`Xd$;yld+%c%d$0BVjs<)u7;oCM4Pnj|h@|VRI4z3fsc5D*Mo)q@@lWO6>j#lxi z_|Q*O2iGjkF_6L zfh%OQ(YtU7$}E@;Z!6}aZDA+eel-=|=U0+3Hf=C^|7U?KN*zp0#l&V`oJjOB4P8n) zVO;Yd78xjotuqTj`tUAz_c{>I>1~4cxB>Vre=<63HXT%Tw6Bzgfcb;;rPWsyCmvLzlcwAdxW~ zQc2VBi-&a?Q41z-O-lv-&N3*THxiGZ5aT7946~NEFirhDycIQ(Xo*sB{Mo~thM z=JRaA+l5$Qogf-({uV5~HTWn8;j9(*3D}BWu-d;XcN$|rZ?De6l=B{>d*}mrzeWrD zdkrNc1G`j>p6mhVZ8C9h;6Zk{q?YZ<6L@?l$TP){8CY(Wha+C-v$P(2Mef}ulcDv6 zFln4T*?)E{%C&EX*#1HnnshJN(AlbP*`$JnsnXK1GK{u#jB^jM_;K7 z7@HFhfhw!fp|lPs_+2L+1}(teMZthc(M)rY0}d<95JiSu7Whzu>v6>vHmUkU?=M4{^SoM6sy2Yn?H<_URSb4X>j2f+ z_i$&09=Z$pueWz(uwm204D3_IF4Xaq#qs!QodY@d`H;FvP zG8TOL1T=UHxGtZ=z*)RYz#HDqPDd>xlCG;o@7<;8T}N*yX?ejy9GoHR-!rh6c&L!a&smmZKVoCc>V=!K)o{*PPufW=kCEn4ZA<*T>^&kYukM z6PUv7JY0f+FK#bmI_*o*?&CtRel?TjP7=9Ks*EMx$?U6;t@}8@ zRn)Oek*h6=#iybzP*zvSR!9gj5M_7C14$jI67UrtJ)6LSItiF}3LVf!Do%8km1E;T z3%;2uCRd9@C2!#AwGUPg4kcrU z&gwK~_kxNc6)$nFVG<5CdB$EEbwi0McSJHzoCVCpn|SlZK^87&iju-NC){6w6aJpgD_79>>l%1|>J+ZNRtTESQtald&Ek&Z z^?6yK8+h-Z%2b>3&?B-@T&^`CwjuH*IDH9 z%|P*|Ns?IDB@=QUT>|B?L2&0r7-q)~5!c^ch%(m)q5p+msEyZftbQ>(2sw$>FSOuI z`!%>B2jgJZL)s$80--mkJ{$RI3dpyJe{twrLG<5^VfsX zXZ>`DnKnp>>K9@h&h~+J*VQm&yB=1+2)LJ!3GdwQLfZFTR#nO$AkMZg^WN*owp$i5 z?e=sCzBUeqtXeF-+3JteWIuvZ;(DxfafRTox52r(h7IFuKqGZJEZ-0g9!=L_^Wg$A zX=Dv(m{h?8IM>c#uCf3R>01k@ zo8q9wUWSfp$->g9yFp2a=i(ERae+j>SY^IGA$Mw_=aJ`Z*!vBlwcE0>Dngf+YMG#9 z!_ca)Ba(6Rwql$xvz%m_4uYzHXeh1f#qK^`M%>Cf^JBt#T3mGo&lx$P;+ejnSmO*) zb;q&O$o`@oQHr3savo?OpF!N>N8l{QT0FhpmEAxS64X*n(nrn2CYe>N=IsQlZM&02 zaY1ctyS^O`tWL&O2{%y4iw6A-GF-oPEBKv|1_d=ocrTox3<&cF>(ha3t8iX%SC8b`D1Rbw#UYOYm563ib^*W4jgfF*EuEJl%W}1{$oxLoLQa&V>%^ z>~^1NPB4dM)8p7Yi@B)pyF$QlGX>*-cW_Bj;M2LU4Ikzng@lU+$({<4@1K3raPEXJRV`J{{&$I8+p1Gb8a9lnw`>t*oUp=I#+bT@t|gfg4` zj^d204*a-W8_9XtNK}{&sSK3oo5y#iF&S6bxY#!W|LF(vIlmMhJWnUR4L)O|=?)y} zwF^U!r(liWE-W_~fi~$~Sm&2=;zfG~K=BK~SKjxLY%Chc4heBBnrC04oz*GWHaP+h zd~U?o28N(_>oQC{vxyx$pDf^PNsuQA26Qkl1@YJE?A@JD_t3(>5bqk--gtpIXGjlu4vW8p3HXHbf!|U2GvIp# z^G*`5a6NcEH)G4nFSGUMe6jMNx4``y96OaOI-Fev#5=wy}!vzF9D%|SJm zU;T*8GP;chF6;5ob$Py5*z+$jeu{QiN{H#~IcP0F2-Mf@67WNsU{L1**ro1_n@c&C zAM(M;dKZa|@pCMB;fdWhAq3yQ0JH3tlJIZ}6+YvL~Nm+4|YRMr)C0oLIoV$ zN94tR8@yRK7qiW($@>ms=sK_o3e*M&vHijsY?s62#9A#}o$4 z{YJ8RI*hJK!qGB`@W!E?>@%>n`Ve4>b1a)cb@U;4zeA7h$!Z2e`(5yn9AX<3GjR2b zhr*d;AuKA4AeqC@67x$V$>z}uV7K!DR%&iTB0Scyl+X&Y-}^CHp2$Et_$vnXOoT7# zXJG5xM`DT1MOF3d+ps*r3~uMIWlxO@$qw0cR%L4mZch)hr6W`D{-cxNbn6M$T2!%5 zgH_<_3R;8Rj}GBu_rh;1eg&yZdW=3ezsZ=B8srN_Xl zkWa8UtA|y+rf|0CA>fZQHKaO!d=sI}Js=MM?+{q-xDU_TSSX&fY< zG^fLmehQ=qMXV{WVWHBmS?#(-pdUJq^%Ksz^SW(k8HT&SspmC#YcT^C6jy<*1g(OHl zsYK77qo8@0GWclafXaeM%$rdGOVkhHJKb}l^6SlLp*)G{X6%40$5N|{9{r)> zL{H4yC&a8(t!I5Ab5UPAMeKO&AX?pSW_$ZChk?uQFdy+CFzN7_NpH5qnxZD`Z($+u z3GN}XGFvd=T`t?6KMkG~_aJ>!wD9g8X?UJ3aMeESK*w!hAT_N3lAbLV-+PrtZf+e7 zn^&e|ll%r~I~54wN2KVQ9>#cc*EF>2DaLiN2k?&83-JOeG2ES8&x)efp>FbWw9VZ~ zj9-j~cSBRz7p3E@cH{??mhB5sn^uXJoyx>U!|gEMwgODM#bU%3V=`$AMm0eHJX5`fwY% z30Onwqom0G97(u-rW;evwG^K?eWPm3#AG=4WF;Psx{ZD$3J#oC#zN(2aJs7w^{<2Q zi|a^ieP1j1V0uvGdmLkT+L9iUYuS~TNqEVB9Fz+Di)!5qAjDz}2BmC({%cKG#Q+A$ zt9(fEgfg(2yFqMauo^vb)3Dy47i@g^90ERBg2~pYxO4tB@;)U?bhzs}lI24Lo@*hO z(NLD8Y>&s&MTz1gFH_+2vUX@QUo5_U_&DYcn*&Lk6EJX2wZP$Rj-PFB3K(j#aOu4q z^eY{WORV)EZJ-;>bDWIK6;YfcXQZ`1~oWhDsY2NbQh~xY#?L2#^CFB z6xo^8aHx9;PP!5S)U%L;?EXMftqCTyMWf-7Y@+e)wrKc`Y*E930yK!PV``G>_*&Kl z?HXP)>)b0WR_8oM-&01tt^$wL9eLRFS%=t8&ITnr0|+ZfXK%yD;0LiH8}!~9jyeQl z(_tpE2>B*9y%<*YWuh0dEtT-Rk2N#!PZuYg$b(5j-rOk-As#7Acn_@Yh5Efq#oxw= z*{SPoY+ShlopVjh7ECRN{vEQIPTm3h=1`07ACI%$$IoMMoh~a|eHlfb&(NLq#QU3k zAmnR*()HE>n7BKMIIi4IWL92=DRQYeCT%BijW@=KGrEv8Z3#$(g$VI8%BVTZoxL8u z9u8<5!Zi6oq?cb3erex?ccvbo*Q7_GjH-$iqcw zW<3tNMoH0=EgpiOq=03iO88{@IBa}78`nK56-j?PP*ohX2H#9^u25GMhc-SbqBe@hdhJ1%>JX46}MWgZlI!>!&{f#@TOlQ8Z3{;t183c#C3ln8Cxj$ zW>-k#DIs=2b{IG!-ep*w@mHGL^!`I7vFDHAz__Y;Itqqs4P4$&euJ_#=PhS z_6j=aBRduLE%t{aySyL~(#4WDV!&{lHTXVkV@EINz(rkMFx=51J~H|}nY(KrCY?S9 zXM5kkuJIN)Y+*J|J-!6CH~K-0QZ1Wsk5{eRoWdmPo?CTf?^r?ZVbP@!V^q}~fV;B@ zNKBGJjR?=m%uYv{QTj3PTKpP2gz7VL--fh|bpicrOGP(^N@2dnJrb8YfmrJjoRbv+ zla-#qc+1lyeba3+x-tgnsSNxu#8njSBgN{Rp5RhX6AX_MbS-+q{EpX?DSZLYZ|ebP z!urGEkG(i`zp$rWtX<%Iq5vtXZBNot~%LA>X))xg1qut#|VRGOTHJ6jZS z$$=vFg03a95y?;>;1m05EJFL|TKM4hUe<6bNmO(4n8--gn9M2iCp|lEg|0>dZdQ#J z%UNIm)1_y^kr`{M%nlV}iE1~P_*D+>ESDl4j=s!vAi>>2e6Q?K3adW4pu@2)Y}n^X zWYD^OtZ&XSD4ekl*Gb=EYxdhg@bvu=$cQ;XNJzRRy zTK4$_54x6(Er|m7LFYhaxr~=SKU()w~v_-^&;4#&*Hp zG2QV?uQ)PS=D2wA+;kjf8%l)PBR-X@3A5MblZ@Czc;WAi(^}T!^rBRV`EEh{1^$gs z{_z6$_<7=Xb}fniK9P8CGlxgTIUvyv1jlqj-~OediO$!=YP-9Ew}ZfsHf=OcF2QI-N?A^rz<%KP2F6@OE&7TRKx;hJIm^aC-3o$tC)NGdFUJ8A~ zhT_aGiFjn>L{Y&hfvf02Z~U^njK%B-$GNK{kX)=LF_lxQB+6!q@)!0%-LrG>!yqU2 zJZUu?xKRL4It_$kZDD^=zW`!xj}}katA^N9cuZq_vrs8=4f2MT8FsQI{hvhewuwYjTHg1f;50bsv z%+1Hi2*ab~)}?*y(;G!FxKISM3mhP)+hQ!3`bHFgAq*=TRx=aG#4aCa!^H9VVh@uZ z5I-tgh#%7ciI*EhPtrF*^zo-`p5Z18FtLDPGgD!J5bxxxAi?fu7UB{gfVMXqL?N8xS&sKpaX@)As0Eh~libRQZWh_h6gMF|!B3f%5h^vMs+AmfC!<_~2 zW_1-)OIQJqib|lqYa>w=_Mn4`#^9YzTX44W7Wx~6|cex>(Yjao>xwtP}?X(77_g;@?^EzOM?pt6)`*yZX zK@X2Cnj_LGGa@Cj$~dCWJh4JzF0>xrXGFCJ~RQx~GWjSsua`+>=xZAe@Oz=x;W*POlK}^vuP`E*damzZO0jVGjk{wupQ^`@@J2Jw&Cg z1m0yw6Qz93LK87X03)4D_^WwqyyI!1^*02 zvGaS9Sa3}~wv}xa)tpGdNfC)4XGn0wU;+Cb?8$(LLC{nXPL2%h1LlEIc*JssXl>{5 znD=2ke48O)DVEM81ykd2R;>*l`Os#S1u7){%|mhe>3oq+lOaaRr=t0p!C~i!K(Of9DXw=7`ZZxb#ysJd{=3~ zJxe1H*A0Zisx`3FP&lHt*eGDbEW>3N#jyLqK`RTpcu?9VaH`EZ3lDrovinAYZ|=XlAxUqR6y!p|I+!GYe^NVrpyaSmcy7 zP$}eQjBGN6w@m}!Twi-^)ww3B>S`g@9KH?BqSE1#fgd|l^R4RnIV~I%w2>vbP|)bq zgPm7A!g5D>fLx0OcwWwj^-sHE_!mhidD9C%8rO(h%OXH8%L^pcRdBJPCF|Se6sRsM z2Hm}-tn$Waz)J;y8&k+ z#zM}52W0Ejt$=ZZ2{9YZ$ucMmN}j; zu~)~Z>tgASPy2=VvvOYhT8|H0rOZVR`+4%w?bLNv96x5O!bj(!-LEpMxMT4NBbnWp`G=0AwKi~BhN(L6=+e7R5&4~V7 ztffysTwDm}WCu|#m40Zvu0Ji3U(8Kp*76RgKI4}90^VG0McIJyw2cnqC0{q-72kgJ zifIl;kNZkY!iQp}#0q-htsI|Jcn+0!YOyWn(|OpWbu_G}5%2K*g{ZE+n9s|RfU#42 z!9v%UhaV~+32!xN&3OsF(qJL3Z68Fv`W(Rr2Zr$_Idef%xf^zJH-^j%9o}uDCO=r$ zC@ym};@?`P(Ovrm4jF~#V0+L24oSGu8CKtLruJj_q~4Rx5EXIbjc%;*Q3~I!;lWqu z+Mvdz+st!QGCz}R1ATIOQMDR38rC+BcXWsqI7yATSTlvj%z7=n6IDcwPaCjpK?;3x z`3`J}HFLR)X? zrqR&TW-86>FH1Gn`BA+^zO-rNSl-Zlh+B?c%k{@4Q?uN9Uh#_3Q#F@x)3z1#baw?9 ze%Fv)P<;aHi#u^HxsslCE93c>zmhdM6GTg_AH#eXXP(~CALHtF2|jTV+{rPgFgJx~ zX^TNS^*ip=v8_7xT@sexK19R5twSq!SL&y{n}?0;NP~~$@rzw!I2|lSH_bl_WqZfa zr)6_-=jSTev_KLX$Gc#7O;6gq`Xb*Lrb{0?iBH1`vo8bp9 zHfhlLY7Km3*)zVjPbU9~UrYPhE zsUG8tZx_O$?E!r04O7%`a^Z)LDbgg{Hq1&F@(C7H!}Z8BB(qN%EWZ#$CkH3u&Dk6H z*=s4-TD*gQN=*}TDh+AOgW1$!ya)Sg(+BU_$5RN*gn$ilEOqTn_Ru|vr@xD(gDb;m zZH+14(|Q^r-VUPr;x#Z8>cYg-y;S!soh!sPZBS-mV`_m&6$IUZXlOA<=`UFLmZEW^FKoNzj(hedsjvSZMwlj?1qL zp>rXBV!XddmxU*3h4R|7qF()(BtM#OplN{)WY46~cw+gg=5k*T7MM_waU8-hc2oj}vBr?`zvBDFlYof{U!a#L3? zXwQxZ$G9MVB=b1Z<>f+7c@l0A@|ofrd%=hvrqnCik#ES!r7M?u(44w$ytzyTosDd` z-OKTKWW*Hyu44-CX*-p-M~=nhr0cYK#4x_9(3OW77V|F^53zlYC-?5DM!oZo@#q<6 znbW;`n9;Ny%*kOWo3dThalu0V;>sgzZIt43w%-S(dAVYdC``0KBZ2$s44~6R<2$;qf*bRXLZbFnw?IZ~cJvnt^q6=N_+MxHC`QtF@o2*j&1z*WHH%1#+Rgbbpov|aYB-Hhubzp!M5*S^!SHhI;Be?AL6Si@Um>+ z_75-79)soR2-)-8+ck@PEk#Jx-Nr{Rjp5GO!?62%J08**>F~6u*na^ zDRroCxs+^ekmp;!c#{b5HjlY-Tl-Fzd``hQ>Wa3lw zy>39ijgW;!0-b}qWIgRV#Gb}}+0SnUS%_C_Sd*bbzU?IY0KWLK0hFr>d+tn+Zti@X zhg|HyYiEA|iwrN?AnidL+HAP{xfFh1e30sY>ddp!R`JHJZg5~yUs{^b4_~a!BsHIf zxJq|1**jju%u8JP0fYYhXp$DIXDiA1PL_1my{-IdcRO07Xw1U{gL&Y?Jf7Nm6N~40 zp^JMQU0>9dRxdw>gEq|>-ffj6-rJT!Pk6258vdNWsxqM}B}4epwf8{BT?8S!cJubn z!8CWcCl6JWryFCt&@Zk5G@;*0?mXfe?>RY@HuPA9-+fyvr@eg3h9^nk)_bY^QusnX z?d@J1j-A*A?R(JBejjdcb;ZuM#VEGU!9CE5lY$u57|1{zth(v@bvL;v38euI5V*t)q_G zBOz_wG5FGc9VN9pQNu-NSnY6oKF~>@ehKvAwT~9jljG}1WvmWA)r;_I&sH$*V8>Ml z?7%HvQ)%N{N)N0SQRloc>Z)x@8=ihA!@RS2>9`R*#3h$6-Y|grwDcy!JQAp>?ENb9@6eK9&$0>=_r5nG=X0_d=EcHis-C62K27cGAf50n#Rgj8yofH zsu8c@z4$Q}o$F1VN1bL@cGh7$TS5#cPNC}Ao48}h2#i{l#??|{>7Duna1QZj)(z!& zQRxJJ47KDfI$k($Qz@nxYQgo9gScXT2flBz5nXP*2EN3%V@hc%3k`u$r&P<`}SIE+ti2}dwW=R^| zT!SML3VGPRi@ckJ-~&yGI zFwfk`;Dm2~&L2(Z^*v+gp$mPf&%;Qb^nC^2dE}Fzt2l1eHIaWmo(^r^| z(rkG3Bmu%!8&Y?*0N%%bwD1kE1YhNigZzyqbo47l*x6f{gPm$YetTCs!L$`-A1H<( z{f#{HW&#YVoJjXS-aovtWfV{Q_86vlcBD__bZAteK0Elyl9jA$1HW)TPVzg_QyO-3 zu+Ivb5?aKgPrK8URt+?i-b^-0+wg8GGW7kn5}41o@eO%fXqjRWm*1FyZB8$F-@0Yw zT;oR!%F7Vd2;Xa^!uXGjk)RJ^RB4+_1y9;l2|Pr~oUaxyq0uX>_*aJ*-nrWu^!{c|P3kxDF10w>yb-RG`+Eb*N2C483`4ktiU-g!gEPq$!OSaHEG7r+2@h_u@(c144$o z?VE=Wo07pO^NINQ`7q3oYa#Di55XomNtAkO09U&;vx<{DnQ2hU7zy7HJBj6!?0)>l z=oN45@2aq`gMy;Gq?+7pzlDL}0|yI*4nGT~3R?0~YF&Q2nkrBT= zCiU|%GvP7mfBKlI(3#KG-|U|ab(dFC)AR`loZ&Zr zcBD^)UsRw^L~u~3pHL@IL0P_&n!-Pd5ylfUkTLpuLaWlqd;1J5J>yq6-YrrzMGn?V4cvw*$VRV z-PNS02mAf5z25}Y`JzD; z^Vfp*{iC2pe-O0azbmL#EZ^Y5SdF?RQK zQ0Ojitfm<%Os0P}=@jJqtI7U0?<0g+J3vitPGDqsuz!Stp8PGubI*7^TorvGtj|I6w6KbiM^gf+t8zpoL0ULSr_>`*}wCTenmw)_Kqek_xNe-?hM za;85EWL2xn{Jij^p83zhk9rnA3qR@&{aN@??}wQ*71ZS|)l?z^!=r*j zgMKbh|8?`n13&(){b9>t|H_ubf3c<2ueKEZoh`|qZTY`m@s#BM!zE6kkG%T7WiAN? z6+xTuQ=5YYCH=TSf4!jg$C>n-iT~NP9~xf&L&MhpO2Z?5(eTJ$HEi>D8Xom$4gar~ za3%Twbj|J~ul;Xn_{Tf&he~aKRq2llqkp~NsWwSLdD2K1hmkf;qkSfgbaEf9rW6nu z8WCvn`+K3=(0}Gz{MSvBK_lJVTpVoN-A22b{IQkatTjc=W;zIQ-q%aQySw_+P&@mE`}^ z*JL00p8unV(ap& zv42?K_16nOG}l*{UXs7hvJSudGE%?$GSY%CBixfwlQjRw)N}j&LHD22@mCZ4zxMYq z!404EtJBy?e)6vlU`P2WYBG^wk$$rkI?7K~lkpFm9~${vPwj=89)GMk?Tkbm0fZ{7UojsAA)?;8ElCC}fxUZfZ$UW^%E_zj7($0QG)xlKn80khcBhfJ zsG=A!pz=})_>ohHnPUPQNV5h|v#H#X`I5ql!iGfJ$fd9W=yna$4->tEL=`27v5*5~ z7Z7tPOqeRHu|VEbSa?el0}wERh-X`VZijOYd|J4Hfsr|vVHyiuEdEtJ+iVt&6)gN9 PA^afxcSE;IsrzjJ8||P@ literal 0 HcmV?d00001 diff --git a/test/TensorFlowNET.Keras.UnitTest/SaveModel/SequentialModelLoad.cs b/test/TensorFlowNET.Keras.UnitTest/SaveModel/SequentialModelLoad.cs new file mode 100644 index 000000000..e778a5a4a --- /dev/null +++ b/test/TensorFlowNET.Keras.UnitTest/SaveModel/SequentialModelLoad.cs @@ -0,0 +1,68 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Tensorflow.Keras.Engine; +using Tensorflow.Keras.Saving.SavedModel; +using Tensorflow.Keras.Losses; +using Tensorflow.Keras.Metrics; +using Tensorflow; +using Tensorflow.Keras.Optimizers; +using static Tensorflow.KerasApi; +using Tensorflow.NumPy; +using static TensorFlowNET.Keras.UnitTest.SaveModel.SequentialModelSave; + +namespace TensorFlowNET.Keras.UnitTest.SaveModel; + +[TestClass] +public class SequentialModelLoad +{ + [TestMethod] + public void SimpleModelFromAutoCompile() + { + var model = keras.models.load_model(@"Assets/simple_model_from_auto_compile"); + model.summary(); + + model.compile(new Adam(0.0001f), new LossesApi().SparseCategoricalCrossentropy(), new string[] { "accuracy" }); + + // check the weights + var kernel1 = np.load(@"Assets/simple_model_from_auto_compile/kernel1.npy"); + var bias0 = np.load(@"Assets/simple_model_from_auto_compile/bias0.npy"); + + Assert.IsTrue(kernel1.Zip(model.TrainableWeights[2].numpy()).All(x => x.First == x.Second)); + Assert.IsTrue(bias0.Zip(model.TrainableWeights[1].numpy()).All(x => x.First == x.Second)); + + var data_loader = new MnistModelLoader(); + var num_epochs = 1; + var batch_size = 8; + + var dataset = data_loader.LoadAsync(new ModelLoadSetting + { + TrainDir = "mnist", + OneHot = false, + ValidationSize = 50000, + }).Result; + + model.fit(dataset.Train.Data, dataset.Train.Labels, batch_size, num_epochs); + } + + [TestMethod] + public void AlexnetFromSequential() + { + new SequentialModelSave().AlexnetFromSequential(); + var model = keras.models.load_model(@"./alexnet_from_sequential"); + model.summary(); + + model.compile(new Adam(0.001f), new LossesApi().SparseCategoricalCrossentropy(from_logits: true), new string[] { "accuracy" }); + + var num_epochs = 1; + var batch_size = 8; + + var dataset = new RandomDataSet(new Shape(227, 227, 3), 16); + + model.fit(dataset.Data, dataset.Labels, batch_size, num_epochs); + } +} diff --git a/test/TensorFlowNET.Keras.UnitTest/SaveModel/SequentialModelTest.cs b/test/TensorFlowNET.Keras.UnitTest/SaveModel/SequentialModelSave.cs similarity index 94% rename from test/TensorFlowNET.Keras.UnitTest/SaveModel/SequentialModelTest.cs rename to test/TensorFlowNET.Keras.UnitTest/SaveModel/SequentialModelSave.cs index 269b9c058..fe9b8b71f 100644 --- a/test/TensorFlowNET.Keras.UnitTest/SaveModel/SequentialModelTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/SaveModel/SequentialModelSave.cs @@ -1,27 +1,21 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; -using Tensorflow.NumPy; -using System; using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +using System.Diagnostics; using Tensorflow; -using static Tensorflow.Binding; -using static Tensorflow.KerasApi; using Tensorflow.Keras; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.Engine; using Tensorflow.Keras.Layers; using Tensorflow.Keras.Losses; -using Tensorflow.Keras.Metrics; using Tensorflow.Keras.Optimizers; -using Tensorflow.Operations; -using System.Diagnostics; +using Tensorflow.NumPy; +using static Tensorflow.Binding; +using static Tensorflow.KerasApi; namespace TensorFlowNET.Keras.UnitTest.SaveModel; [TestClass] -public class SequentialModelTest +public class SequentialModelSave { [TestMethod] public void SimpleModelFromAutoCompile() @@ -63,6 +57,8 @@ public void SimpleModelFromSequential() keras.layers.Softmax(1) }); + model.summary(); + model.compile(new Adam(0.001f), new LossesApi().SparseCategoricalCrossentropy(), new string[] { "accuracy" }); var data_loader = new MnistModelLoader(); @@ -82,7 +78,7 @@ public void SimpleModelFromSequential() } [TestMethod] - public void AlexModelFromSequential() + public void AlexnetFromSequential() { Model model = KerasApi.keras.Sequential(new List() { @@ -116,7 +112,7 @@ public void AlexModelFromSequential() keras.layers.Softmax(1) }); - model.compile(new Adam(0.001f), new LossesApi().SparseCategoricalCrossentropy(from_logits:true), new string[] { "accuracy" }); + model.compile(new Adam(0.001f), new LossesApi().SparseCategoricalCrossentropy(from_logits: true), new string[] { "accuracy" }); var num_epochs = 1; var batch_size = 8; @@ -125,7 +121,7 @@ public void AlexModelFromSequential() model.fit(dataset.Data, dataset.Labels, batch_size, num_epochs); - model.save("./pb_alex_sequential", save_format: "tf"); + model.save("./alexnet_from_sequential", save_format: "tf"); // The saved model can be test with the following python code: #region alexnet_python_code diff --git a/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj b/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj index c9020f7b4..bcd52c228 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj +++ b/test/TensorFlowNET.Keras.UnitTest/Tensorflow.Keras.UnitTest.csproj @@ -27,4 +27,28 @@ + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + From 9aa8f758f402cb108fabd7be1c043996acadf859 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Thu, 2 Mar 2023 21:57:48 -0600 Subject: [PATCH 48/52] Fix sparse_categorical_crossentropy. --- src/TensorFlowNET.Keras/BackendImpl.cs | 5 ++--- test/TensorFlowNET.Keras.UnitTest/MultiThreadsTest.cs | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/TensorFlowNET.Keras/BackendImpl.cs b/src/TensorFlowNET.Keras/BackendImpl.cs index c49fc1409..01aa59b9a 100644 --- a/src/TensorFlowNET.Keras/BackendImpl.cs +++ b/src/TensorFlowNET.Keras/BackendImpl.cs @@ -307,9 +307,8 @@ public Tensor sparse_categorical_crossentropy(Tensor target, Tensor output, bool var update_shape = target_rank > -1 && output_rank > -1 && target_rank != output_rank - 1; if (update_shape) { - /*var target = flatten(target); - output = tf.reshape(output, [-1, output_shape[-1]]);*/ - throw new NotImplementedException(""); + target = tf.reshape(target, -1); + output = tf.reshape(output, (-1, output.shape[-1])); } if (ignore_class.HasValue) diff --git a/test/TensorFlowNET.Keras.UnitTest/MultiThreadsTest.cs b/test/TensorFlowNET.Keras.UnitTest/MultiThreadsTest.cs index ffe3d6f43..555154d70 100644 --- a/test/TensorFlowNET.Keras.UnitTest/MultiThreadsTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/MultiThreadsTest.cs @@ -8,7 +8,7 @@ namespace TensorFlowNET.Keras.UnitTest { - [TestClass, Ignore] + [TestClass] public class MultiThreads { [TestMethod] From afa269699fb1ebff21f5ba3e25ac0a816e1e88f6 Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 3 Mar 2023 07:37:41 -0600 Subject: [PATCH 49/52] MultiThreads Failed on MacOS. --- test/TensorFlowNET.Keras.UnitTest/MultiThreadsTest.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/TensorFlowNET.Keras.UnitTest/MultiThreadsTest.cs b/test/TensorFlowNET.Keras.UnitTest/MultiThreadsTest.cs index 555154d70..2d487087d 100644 --- a/test/TensorFlowNET.Keras.UnitTest/MultiThreadsTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/MultiThreadsTest.cs @@ -11,7 +11,7 @@ namespace TensorFlowNET.Keras.UnitTest [TestClass] public class MultiThreads { - [TestMethod] + [TestMethod, Ignore("Failed on MacOS")] public void Test1() { //Arrange @@ -26,7 +26,7 @@ public void Test1() } - [TestMethod] + [TestMethod, Ignore("Failed on MacOS")] public void Test2() { //Arrange @@ -40,7 +40,7 @@ public void Test2() } - [TestMethod] + [TestMethod, Ignore("Failed on MacOS")] public void Test3Multithreading() { //Arrange From b8fd21c09416135f187c03295fa5433c5e61982a Mon Sep 17 00:00:00 2001 From: Yaohui Liu Date: Fri, 3 Mar 2023 20:58:12 +0800 Subject: [PATCH 50/52] Fix issue #760 --- .../Operations/gen_array_ops.cs | 2 +- test/TensorFlowNET.Keras.UnitTest/Gradient.cs | 76 +++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 test/TensorFlowNET.Keras.UnitTest/Gradient.cs diff --git a/src/TensorFlowNET.Core/Operations/gen_array_ops.cs b/src/TensorFlowNET.Core/Operations/gen_array_ops.cs index 794c32673..93a54af00 100644 --- a/src/TensorFlowNET.Core/Operations/gen_array_ops.cs +++ b/src/TensorFlowNET.Core/Operations/gen_array_ops.cs @@ -269,7 +269,7 @@ public static (Tensor, Tensor) unique(Tensor x, TF_DataType out_idx = TF_DataTyp public static Tensor[] unpack(Tensor value, int num, int axis = 0, string name = null) => tf.Context.ExecuteOp("Unpack", name, new ExecuteOpArgs(value, num) - .SetAttributes(new { axis })); + .SetAttributes(new { axis, num })); public static Tensor where(Tensor condition, string name = null) { diff --git a/test/TensorFlowNET.Keras.UnitTest/Gradient.cs b/test/TensorFlowNET.Keras.UnitTest/Gradient.cs new file mode 100644 index 000000000..159800e10 --- /dev/null +++ b/test/TensorFlowNET.Keras.UnitTest/Gradient.cs @@ -0,0 +1,76 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System.Linq; +using Tensorflow; +using Tensorflow.Keras.Engine; +using static Tensorflow.Binding; +using static Tensorflow.KerasApi; +using Tensorflow.NumPy; + +namespace TensorFlowNET.Keras.UnitTest; + +[TestClass] +public class GradientTest +{ + public Model get_actor(int num_states) + { + var inputs = keras.layers.Input(shape: num_states); + var outputs = keras.layers.Dense(1, activation: keras.activations.Tanh).Apply(inputs); + + Model model = keras.Model(inputs, outputs); + + return model; + } + + public Model get_critic(int num_states, int num_actions) + { + // State as input + var state_input = keras.layers.Input(shape: num_states); + + // Action as input + var action_input = keras.layers.Input(shape: num_actions); + + var concat = keras.layers.Concatenate(axis: 1).Apply(new Tensors(state_input, action_input)); + + var outputs = keras.layers.Dense(1).Apply(concat); + + Model model = keras.Model(new Tensors(state_input, action_input), outputs); + model.summary(); + + return model; + } + + [TestMethod] + public void GetGradient_Test() + { + var numStates = 3; + var numActions = 1; + var batchSize = 64; + var gamma = 0.99f; + + var target_actor_model = get_actor(numStates); + var target_critic_model = get_critic(numStates, numActions); + var critic_model = get_critic(numStates, numActions); + + Tensor state_batch = tf.convert_to_tensor(np.zeros((batchSize, numStates)), TF_DataType.TF_FLOAT); + Tensor action_batch = tf.convert_to_tensor(np.zeros((batchSize, numActions)), TF_DataType.TF_FLOAT); + Tensor reward_batch = tf.convert_to_tensor(np.zeros((batchSize, 1)), TF_DataType.TF_FLOAT); + Tensor next_state_batch = tf.convert_to_tensor(np.zeros((batchSize, numStates)), TF_DataType.TF_FLOAT); + + using (var tape = tf.GradientTape()) + { + var target_actions = target_actor_model.Apply(next_state_batch, training: true); + var target_critic_value = target_critic_model.Apply(new Tensors(new Tensor[] { next_state_batch, target_actions }), training: true); + + var y = reward_batch + tf.multiply(gamma, target_critic_value); + + var critic_value = critic_model.Apply(new Tensors(new Tensor[] { state_batch, action_batch }), training: true); + + var critic_loss = math_ops.reduce_mean(math_ops.square(y - critic_value)); + + var critic_grad = tape.gradient(critic_loss, critic_model.TrainableVariables); + + Assert.IsNotNull(critic_grad); + Assert.IsNotNull(critic_grad.First()); + } + } +} \ No newline at end of file From 559d471407b437abe262d1a3edfe4dd20269415d Mon Sep 17 00:00:00 2001 From: Yaohui Liu Date: Fri, 3 Mar 2023 22:35:41 +0800 Subject: [PATCH 51/52] Align keras.Input with tensorflow python. --- .../Keras/Layers/ILayersApi.cs | 10 +++- src/TensorFlowNET.Keras/KerasInterface.cs | 40 +++++---------- src/TensorFlowNET.Keras/Layers/LayersApi.cs | 51 ++++++++++++++++--- .../Layers/AttentionTest.cs | 2 +- 4 files changed, 65 insertions(+), 38 deletions(-) diff --git a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs index 9fcd0d70f..6b2c38c32 100644 --- a/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs +++ b/src/TensorFlowNET.Core/Keras/Layers/ILayersApi.cs @@ -1,4 +1,5 @@ using System; +using Tensorflow.Framework.Models; using Tensorflow.NumPy; using static Google.Protobuf.Reflection.FieldDescriptorProto.Types; @@ -133,11 +134,16 @@ public ILayer EinsumDense(string equation, public ILayer GlobalMaxPooling1D(string data_format = "channels_last"); public ILayer GlobalMaxPooling2D(string data_format = "channels_last"); - public Tensors Input(Shape shape, + public Tensors Input(Shape shape = null, int batch_size = -1, string name = null, + TF_DataType dtype = TF_DataType.DtInvalid, bool sparse = false, - bool ragged = false); + Tensor tensor = null, + bool ragged = false, + TypeSpec type_spec = null, + Shape batch_input_shape = null, + Shape batch_shape = null); public ILayer InputLayer(Shape input_shape, string name = null, bool sparse = false, diff --git a/src/TensorFlowNET.Keras/KerasInterface.cs b/src/TensorFlowNET.Keras/KerasInterface.cs index e0d148cef..2bde713c0 100644 --- a/src/TensorFlowNET.Keras/KerasInterface.cs +++ b/src/TensorFlowNET.Keras/KerasInterface.cs @@ -12,6 +12,7 @@ using Tensorflow.Keras.Optimizers; using Tensorflow.Keras.Utils; using System.Threading; +using Tensorflow.Framework.Models; namespace Tensorflow.Keras { @@ -66,33 +67,16 @@ public Functional Model(Tensors inputs, Tensors outputs, string name = null) /// If set, the layer will not create a placeholder tensor. /// /// - public Tensor Input(Shape shape = null, - int batch_size = -1, - Shape batch_input_shape = null, - TF_DataType dtype = TF_DataType.DtInvalid, - string name = null, - bool sparse = false, - bool ragged = false, - Tensor tensor = null) - { - if (batch_input_shape != null) - shape = batch_input_shape.dims.Skip(1).ToArray(); - - var args = new InputLayerArgs - { - Name = name, - InputShape = shape, - BatchInputShape = batch_input_shape, - BatchSize = batch_size, - DType = dtype, - Sparse = sparse, - Ragged = ragged, - InputTensor = tensor - }; - - var layer = new InputLayer(args); - - return layer.InboundNodes[0].Outputs; - } + public Tensors Input(Shape shape = null, + int batch_size = -1, + string name = null, + TF_DataType dtype = TF_DataType.DtInvalid, + bool sparse = false, + Tensor tensor = null, + bool ragged = false, + TypeSpec type_spec = null, + Shape batch_input_shape = null, + Shape batch_shape = null) => keras.layers.Input(shape, batch_size, name, + dtype, sparse, tensor, ragged, type_spec, batch_input_shape, batch_shape); } } diff --git a/src/TensorFlowNET.Keras/Layers/LayersApi.cs b/src/TensorFlowNET.Keras/Layers/LayersApi.cs index 0d71b2713..cf689edf1 100644 --- a/src/TensorFlowNET.Keras/Layers/LayersApi.cs +++ b/src/TensorFlowNET.Keras/Layers/LayersApi.cs @@ -1,4 +1,5 @@ using System; +using Tensorflow.Framework.Models; using Tensorflow.Keras.ArgsDefinition; using Tensorflow.Keras.ArgsDefinition.Core; using Tensorflow.Keras.ArgsDefinition.Rnn; @@ -471,20 +472,56 @@ public ILayer Flatten(string data_format = null) /// In this case, values of 'None' in the 'shape' argument represent ragged dimensions. For more information about RaggedTensors, see this guide. /// /// A tensor. - public Tensors Input(Shape shape, + public Tensors Input(Shape shape = null, int batch_size = -1, string name = null, + TF_DataType dtype = TF_DataType.DtInvalid, bool sparse = false, - bool ragged = false) + Tensor tensor = null, + bool ragged = false, + TypeSpec type_spec = null, + Shape batch_input_shape = null, + Shape batch_shape = null) { - var input_layer = new InputLayer(new InputLayerArgs + if(sparse && ragged) + { + throw new ValueError("Cannot set both `sparse` and `ragged` to `true` in a Keras `Input`."); + } + + InputLayerArgs input_layer_config = new() { - InputShape = shape, - BatchSize= batch_size, Name = name, + DType = dtype, Sparse = sparse, - Ragged = ragged - }); + Ragged = ragged, + InputTensor = tensor, + // skip the `type_spec` + }; + + if(shape is not null && batch_input_shape is not null) + { + throw new ValueError("Only provide the `shape` OR `batch_input_shape` argument " + + "to Input, not both at the same time."); + } + + if(batch_input_shape is null && shape is null && tensor is null && type_spec is null) + { + throw new ValueError("Please provide to Input a `shape` or a `tensor` or a `type_spec` argument. Note that " + + "`shape` does not include the batch dimension."); + } + + if(batch_input_shape is not null) + { + shape = batch_input_shape["1:"]; + input_layer_config.BatchInputShape = batch_input_shape; + } + else + { + input_layer_config.BatchSize = batch_size; + input_layer_config.InputShape = shape; + } + + var input_layer = new InputLayer(input_layer_config); return input_layer.InboundNodes[0].Outputs; } diff --git a/test/TensorFlowNET.Keras.UnitTest/Layers/AttentionTest.cs b/test/TensorFlowNET.Keras.UnitTest/Layers/AttentionTest.cs index 02298ce81..e5987f298 100644 --- a/test/TensorFlowNET.Keras.UnitTest/Layers/AttentionTest.cs +++ b/test/TensorFlowNET.Keras.UnitTest/Layers/AttentionTest.cs @@ -158,7 +158,7 @@ public void test_masked_attention() var value = keras.Input(shape: (2, 8)); var mask_tensor = keras.Input(shape:(4, 2)); var attention_layer = keras.layers.MultiHeadAttention(num_heads: 2, key_dim: 2); - attention_layer.Apply(new[] { query, value, mask_tensor }); + attention_layer.Apply(new Tensor[] { query, value, mask_tensor }); var from_data = 10 * np.random.randn(batch_size, 4, 8); var to_data = 10 * np.random.randn(batch_size, 2, 8); From 016294d33fe7a8600561fa14e659f7651bbf7fde Mon Sep 17 00:00:00 2001 From: Haiping Chen Date: Fri, 3 Mar 2023 10:35:58 -0600 Subject: [PATCH 52/52] Seperate SafeCheckpointReaderHandle. --- .../Checkpoint/CheckpointReader.cs | 137 +++++++----------- .../Checkpoint/SafeCheckpointReaderHandle.cs | 21 +++ 2 files changed, 74 insertions(+), 84 deletions(-) create mode 100644 src/TensorFlowNET.Core/Checkpoint/SafeCheckpointReaderHandle.cs diff --git a/src/TensorFlowNET.Core/Checkpoint/CheckpointReader.cs b/src/TensorFlowNET.Core/Checkpoint/CheckpointReader.cs index 0cc8e5fbd..ffefe3128 100644 --- a/src/TensorFlowNET.Core/Checkpoint/CheckpointReader.cs +++ b/src/TensorFlowNET.Core/Checkpoint/CheckpointReader.cs @@ -1,100 +1,69 @@ -using Tensorflow.Util; +namespace Tensorflow.Checkpoint; -namespace Tensorflow.Checkpoint +public class CheckpointReader { - sealed class SafeCheckpointReaderHandle : SafeTensorflowHandle - { - public SafeCheckpointReaderHandle(): base() - { - - } - public SafeCheckpointReaderHandle(IntPtr handle): base(handle) - { + private SafeCheckpointReaderHandle _handle; + public Dictionary VariableToDataTypeMap { get; set; } + public Dictionary VariableToShapeMap { get; set; } - } - - protected override bool ReleaseHandle() - { - c_api.TF_DeleteCheckpointReader(handle); - SetHandle(IntPtr.Zero); - return true; - } - } - public class CheckpointReader + public CheckpointReader(string filename) { - private SafeCheckpointReaderHandle _handle; - public Dictionary VariableToDataTypeMap { get; set; } - public Dictionary VariableToShapeMap { get; set; } - - public CheckpointReader(string filename) - { - Status status = new Status(); - _handle = c_api.TF_NewCheckpointReader(filename, status.Handle); - status.Check(true); - ReadAllShapeAndType(); - } + Status status = new Status(); + VariableToDataTypeMap = new Dictionary(); + VariableToShapeMap = new Dictionary(); + _handle = c_api.TF_NewCheckpointReader(filename, status.Handle); + status.Check(true); + ReadAllShapeAndType(); + } - public int HasTensor(string name) - { - return c_api.TF_CheckpointReaderHasTensor(_handle, name); - } + public int HasTensor(string name) + => c_api.TF_CheckpointReaderHasTensor(_handle, name); - ///

- /// Get the variable name. - /// - /// - /// - public string GetVariable(int index) - { - return c_api.StringPiece(c_api.TF_CheckpointReaderGetVariable(_handle, index)); - } + /// + /// Get the variable name. + /// + /// + /// + public string GetVariable(int index) + => c_api.StringPiece(c_api.TF_CheckpointReaderGetVariable(_handle, index)); - public int Size() - { - return c_api.TF_CheckpointReaderSize(_handle); - } + public int Size() + => c_api.TF_CheckpointReaderSize(_handle); - public TF_DataType GetVariableDataType(string name) - { - return c_api.TF_CheckpointReaderGetVariableDataType(_handle, name); - } + public TF_DataType GetVariableDataType(string name) + => c_api.TF_CheckpointReaderGetVariableDataType(_handle, name); - public Shape GetVariableShape(string name) - { - int num_dims = GetVariableNumDims(name); - long[] dims = new long[num_dims]; - Status status = new Status(); - c_api.TF_CheckpointReaderGetVariableShape(_handle, name, dims, num_dims, status.Handle); - status.Check(true); - return new Shape(dims); - } + public Shape GetVariableShape(string name) + { + int num_dims = GetVariableNumDims(name); + long[] dims = new long[num_dims]; + Status status = new Status(); + c_api.TF_CheckpointReaderGetVariableShape(_handle, name, dims, num_dims, status.Handle); + status.Check(true); + return new Shape(dims); + } - public int GetVariableNumDims(string name) - { - return c_api.TF_CheckpointReaderGetVariableNumDims(_handle, name); - } + public int GetVariableNumDims(string name) + => c_api.TF_CheckpointReaderGetVariableNumDims(_handle, name); - public unsafe Tensor GetTensor(string name, TF_DataType dtype = TF_DataType.DtInvalid) - { - Status status = new Status(); - var tensor = c_api.TF_CheckpointReaderGetTensor(_handle, name, status.Handle); - status.Check(true); - return new Tensor(tensor); - } + public unsafe Tensor GetTensor(string name, TF_DataType dtype = TF_DataType.DtInvalid) + { + Status status = new Status(); + var tensor = c_api.TF_CheckpointReaderGetTensor(_handle, name, status.Handle); + status.Check(true); + return new Tensor(tensor); + } - private void ReadAllShapeAndType() + private void ReadAllShapeAndType() + { + int size = Size(); + for(int i = 0; i < size; i++) { - VariableToDataTypeMap = new Dictionary(); - VariableToShapeMap = new Dictionary(); - int size = Size(); - for(int i = 0; i < size; i++) - { - var name = GetVariable(i); - var shape = GetVariableShape(name); - var dtype = GetVariableDataType(name); - VariableToDataTypeMap[name] = dtype; - VariableToShapeMap[name] = shape; - } + var name = GetVariable(i); + var shape = GetVariableShape(name); + var dtype = GetVariableDataType(name); + VariableToDataTypeMap[name] = dtype; + VariableToShapeMap[name] = shape; } } } diff --git a/src/TensorFlowNET.Core/Checkpoint/SafeCheckpointReaderHandle.cs b/src/TensorFlowNET.Core/Checkpoint/SafeCheckpointReaderHandle.cs new file mode 100644 index 000000000..674e83512 --- /dev/null +++ b/src/TensorFlowNET.Core/Checkpoint/SafeCheckpointReaderHandle.cs @@ -0,0 +1,21 @@ +using Tensorflow.Util; + +namespace Tensorflow.Checkpoint; + +public sealed class SafeCheckpointReaderHandle : SafeTensorflowHandle +{ + private SafeCheckpointReaderHandle() : base () + { + } + + public SafeCheckpointReaderHandle(IntPtr handle) : base(handle) + { + } + + protected override bool ReleaseHandle() + { + c_api.TF_DeleteCheckpointReader(handle); + SetHandle(IntPtr.Zero); + return true; + } +}