feat(validation): add day of the month add assertions and error handling

Gandalfdore · Gandalfdore · commit fa88c2411e09 · 2025-01-16T19:43:01.000+02:00
diff --git a/.gitignore b/.gitignore
@@ -161,3 +161,6 @@ cython_debug/
 .DS_Store
 kdp/data/fake_data.csv
 *.json
+
+# Ignore all contents of my_tests folder
+my_tests/*
diff --git a/README.md b/README.md
@@ -61,7 +61,7 @@ features_specs = {
 # INSTANTIATE THE PREPROCESSING MODEL with your data
 ppr = PreprocessingModel(
     path_data="data/my_data.csv",
-    features_specs=features_spec,
+    features_specs=features_specs,
 )
 # construct the preprocessing pipelines
 ppr.build_preprocessor()
diff --git a/docs/contributing.md b/docs/contributing.md
@@ -59,7 +59,7 @@ If you wan't to request a new feature or you have detected an issue, please use
 
 - [x] Create your first Merge Request (MR) as soon as possible.
 
-  > Merge requests will be responsible for semantic-release storytelling and so use them wisely! The changelog report generated automatically will be based on your commits merged into main branch and should cover all the thins you did for the project, as an example:
+  > Merge requests will be responsible for semantic-release storytelling and so use them wisely! The changelog report generated automatically will be based on your commits merged into main branch and should cover all the things you did for the project, as an example:
 
 - [x] Separate your merge requests based on LABEL or functionality if you are working on `feat` label
 
@@ -75,7 +75,7 @@ If you wan't to request a new feature or you have detected an issue, please use
 
         `{LABEL}(KDP): {message}`
 
-- [x] Use small Merge Requests but do them more ofthen < 400 ligns for quicker and simple review and not the whole project !
+- [x] Use small Merge Requests but do them more ofthen < 400 lines for quicker and simple review and not the whole project !
 
 - [x] Ask for a Code Review !
 
diff --git a/docs/features.md b/docs/features.md
@@ -205,7 +205,7 @@ Example cross feature between INTEGER_CATEGORICAL and STRING_CATEGORICAL:
 
 ## 📆 Date Features
 
-You can even process string encoded date features (format: 'YYYY-MM-DD'):
+You can even process string encoded date features (format: 'YYYY-MM-DD' or 'YYYY/MM/DD'):
 
 === "🔧 Using FeatureType"
 
diff --git a/kdp/custom_layers.py b/kdp/custom_layers.py
@@ -90,6 +90,7 @@ def __init__(self, date_format: str = "YYYY-MM-DD", **kwargs) -> None:
 
         Args:
             date_format (str): format of the string encoded date to parse.
+                Supported formats: YYYY-MM-DD, YYYY/MM/DD
             kwargs (dict): other params to pass to the class.
         """
         super().__init__(**kwargs)
@@ -103,31 +104,55 @@ def call(self, inputs: tf.Tensor) -> tf.Tensor:
             inputs (tf.Tensor): Tensor with input data.
 
         Returns:
-            tf.Tensor: processed date tensor with all cyclic components.
+            tf.Tensor: processed date tensor with all components [year, month, day_of_month, day_of_week].
         """
 
         def parse_date(date_str: str) -> tf.Tensor:
+            # Handle missing/invalid dates
+            is_valid = tf.strings.regex_full_match(date_str, r"^\d{1,4}[-/]\d{1,2}[-/]\d{1,2}$")
+            tf.debugging.assert_equal(
+                is_valid,
+                True,
+                message="Invalid date format. Expected YYYY-MM-DD or YYYY/MM/DD",
+            )
+
+            # First, standardize the separator to '-' in case of YYYY/MM/DD format
+            date_str = tf.strings.regex_replace(date_str, "/", "-")
+
             parts = tf.strings.split(date_str, "-")
             year = tf.strings.to_number(parts[0], out_type=tf.int32)
             month = tf.strings.to_number(parts[1], out_type=tf.int32)
-            day = tf.strings.to_number(parts[2], out_type=tf.int32)
+            day_of_month = tf.strings.to_number(parts[2], out_type=tf.int32)
+
+            # Validate date components
+            # Validate year is in reasonable range
+            tf.debugging.assert_greater_equal(year, 1000, message="Year must be >= 1000")
+            tf.debugging.assert_less_equal(year, 2200, message="Year must be <= 2200")
+
+            # Validate month is between 1-12
+            tf.debugging.assert_greater_equal(month, 1, message="Month must be >= 1")
+            tf.debugging.assert_less_equal(month, 12, message="Month must be <= 12")
+
+            # Validate day is between 1-31
+            tf.debugging.assert_greater_equal(day_of_month, 1, message="Day must be >= 1")
+            tf.debugging.assert_less_equal(day_of_month, 31, message="Day must be <= 31")
 
             # Calculate day of week using Zeller's congruence
             y = tf.where(month < 3, year - 1, year)
             m = tf.where(month < 3, month + 12, month)
             k = y % 100
             j = y // 100
-            h = (day + ((13 * (m + 1)) // 5) + k + (k // 4) + (j // 4) - (2 * j)) % 7
+            h = (day_of_month + ((13 * (m + 1)) // 5) + k + (k // 4) + (j // 4) - (2 * j)) % 7
             day_of_week = tf.where(h == 0, 6, h - 1)  # Adjust to 0-6 range where 0 is Monday
 
-            return tf.stack([year, month, day_of_week])
+            return tf.stack([year, month, day_of_month, day_of_week])
 
         parsed_dates = tf.map_fn(parse_date, tf.squeeze(inputs), fn_output_signature=tf.int32)
         return parsed_dates
 
     def compute_output_shape(self, input_shape: int) -> int:
         """Getting output shape."""
-        return tf.TensorShape([input_shape[0], 3])
+        return tf.TensorShape([input_shape[0], 4])  # Changed to 4 components
 
     def get_config(self) -> dict:
         """Saving configuration."""
@@ -171,14 +196,14 @@ def cyclic_encoding(self, value: tf.Tensor, period: float) -> tuple[tf.Tensor, t
 
     @tf.function
     def call(self, inputs: tf.Tensor) -> tf.Tensor:
-        """Splits the date into 3 components: year, month and day and
+        """Splits the date into 4 components: year, month, day and day of the week and
         encodes it into sin and cos cyclical projections.
 
         Args:
-            inputs (tf.Tensor): input data.
+            inputs (tf.Tensor): input data [year, month, day_of_month, day_of_week].
 
         Returns:
-            (tf.Tensor): cyclically encoded data (sin and cos).
+            tf.Tensor: cyclically encoded data (sin and cos components).
         """
         # Reshape input if necessary
         input_shape = tf.shape(inputs)
@@ -188,19 +213,22 @@ def call(self, inputs: tf.Tensor) -> tf.Tensor:
         # Extract features
         year = inputs[:, 0]
         month = inputs[:, 1]
-        day_of_week = inputs[:, 2]
+        day_of_month = inputs[:, 2]  # New: day of month
+        day_of_week = inputs[:, 3]  # Now at index 3
 
-        # Cyclical encoding
+        # Convert to float
         year_float = tf.cast(year, tf.float32)
         month_float = tf.cast(month, tf.float32)
+        day_of_month_float = tf.cast(day_of_month, tf.float32)
         day_of_week_float = tf.cast(day_of_week, tf.float32)
 
         # Ensure inputs are in the correct range
         year_float = self.normalize_year(year_float)
 
-        # Encode each feature
+        # Encode each feature in cyclinc projections
         year_sin, year_cos = self.cyclic_encoding(year_float, period=1.0)
         month_sin, month_cos = self.cyclic_encoding(month_float, period=12.0)
+        day_of_month_sin, day_of_month_cos = self.cyclic_encoding(day_of_month_float, period=31.0)
         day_of_week_sin, day_of_week_cos = self.cyclic_encoding(day_of_week_float, period=7.0)
 
         encoded = tf.stack(
@@ -209,6 +237,8 @@ def call(self, inputs: tf.Tensor) -> tf.Tensor:
                 year_cos,
                 month_sin,
                 month_cos,
+                day_of_month_sin,  # New
+                day_of_month_cos,  # New
                 day_of_week_sin,
                 day_of_week_cos,
             ],
@@ -219,7 +249,7 @@ def call(self, inputs: tf.Tensor) -> tf.Tensor:
 
     def compute_output_shape(self, input_shape: int) -> int:
         """Getting output shape."""
-        return tf.TensorShape([input_shape[0], 6])
+        return tf.TensorShape([input_shape[0], 8])  # Changed to 8 for 4 features * 2 components each
 
     def get_config(self) -> dict:
         """Returns the configuration of the layer as a dictionary."""

Original file line number	Diff line number	Diff line change
`@@ -61,7 +61,7 @@ features_specs = {`
`61`	`61`	`# INSTANTIATE THE PREPROCESSING MODEL with your data`
`62`	`62`	`ppr = PreprocessingModel(`
`63`	`63`	`path_data="data/my_data.csv",`
`64`		`- features_specs=features_spec,`
	`64`	`+ features_specs=features_specs,`
`65`	`65`	`)`
`66`	`66`	`# construct the preprocessing pipelines`
`67`	`67`	`ppr.build_preprocessor()`