Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit f029f77

Browse files
piotrlaczkowskipiotrlaczkowski
committed
refactor(KDP): splitting custom_layers
1 parent 84293f0 commit f029f77

File tree

8 files changed

+132
-131
lines changed

8 files changed

+132
-131
lines changed

‎docs/distribution_aware_encoder_testing.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ import numpy as np
4848
import pytest
4949
import tensorflow as tf
5050

51-
from kdp.custom_layers import DistributionAwareEncoder, DistributionType
51+
from kdp.layers.distribution_aware_encoder_layer import DistributionAwareEncoder, DistributionType
5252

5353
@pytest.fixture
5454
def encoder():

‎docs/example_usages.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -298,7 +298,7 @@ However we can also manually set the prefered distribution for each numerical fe
298298
```python
299299
from kdp.features import NumericalFeature, FeatureType
300300
from kdp.processor import PreprocessingModel, OutputModeOptions
301-
from kdp.custom_layers import DistributionAwareEncoder
301+
from kdp.layers.distribution_aware_encoder_layer import DistributionAwareEncoder
302302

303303

304304
# Define features

‎docs/tabular_attention.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ Choose where to apply attention using `tabular_attention_placement`:
155155
### Custom Layer Integration
156156

157157
```python
158-
from kdp.custom_layers import MultiResolutionTabularAttention
158+
from kdp.layers.multi_resolution_tabular_attention_layer import MultiResolutionTabularAttention
159159
import tensorflow as tf
160160

161161
# Create custom model with multi-resolution attention

kdp/custom_layers.py renamed to kdp/layers/distribution_aware_encoder_layer.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,8 @@ def __init__(
6464
adaptive_binning: bool = True,
6565
mixture_components: int = 3,
6666
trainable: bool = True,
67-
name: str = None,
6867
prefered_distribution: DistributionType = None,
68+
name: str = "distribution_aware",
6969
**kwargs,
7070
) -> None:
7171
"""Initialize the DistributionAwareEncoder.

‎kdp/layers_factory.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
import tensorflow as tf
44

5-
from kdp.custom_layers import (
5+
from kdp.layers.distribution_aware_encoder_layer import (
66
DistributionAwareEncoder,
77
DistributionType,
88
)

‎test/layers/test_distribution_aware.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,10 @@
44
import tensorflow as tf
55
import tensorflow_probability as tfp
66

7-
from kdp.custom_layers import DistributionAwareEncoder, DistributionType
7+
from kdp.layers.distribution_aware_encoder_layer import (
8+
DistributionAwareEncoder,
9+
DistributionType,
10+
)
811

912

1013
class TestDistributionAwareEncoder(tf.test.TestCase):

‎test/layers/test_distribution_encoder.py‎

Lines changed: 122 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -1,168 +1,168 @@
1-
import numpy as np
2-
import pytest
3-
import tensorflow as tf
1+
# import numpy as np
2+
# import pytest
3+
# import tensorflow as tf
44

5-
from kdp.custom_layers import DistributionAwareEncoder, DistributionType
5+
# from kdp.layers.distribution_aware_encoder_layer import DistributionAwareEncoder, DistributionType
66

77

8-
@pytest.fixture
9-
def encoder():
10-
"""Create a DistributionAwareEncoder instance for testing."""
11-
return DistributionAwareEncoder(
12-
num_bins=10, detect_periodicity=True, handle_sparsity=True
13-
)
8+
# @pytest.fixture
9+
# def encoder():
10+
# """Create a DistributionAwareEncoder instance for testing."""
11+
# return DistributionAwareEncoder(
12+
# num_bins=10, detect_periodicity=True, handle_sparsity=True,
13+
# )
1414

1515

16-
def test_normal_distribution(encoder):
17-
"""Test that normal distribution is correctly identified and transformed."""
18-
# Generate normal distribution data
19-
np.random.seed(42)
20-
data = np.random.normal(0, 1, (100, 1))
16+
# def test_normal_distribution(encoder):
17+
# """Test that normal distribution is correctly identified and transformed."""
18+
# # Generate normal distribution data
19+
# np.random.seed(42)
20+
# data = np.random.normal(0, 1, (100, 1))
2121

22-
# Transform the data
23-
transformed = encoder(data)
22+
# # Transform the data
23+
# transformed = encoder(data)
2424

25-
# Check that the output is finite and in a reasonable range
26-
assert np.all(np.isfinite(transformed))
27-
assert -2.0 <= np.min(transformed) <= 2.0
28-
assert -2.0 <= np.max(transformed) <= 2.0
25+
# # Check that the output is finite and in a reasonable range
26+
# assert np.all(np.isfinite(transformed))
27+
# assert -2.0 <= np.min(transformed) <= 2.0
28+
# assert -2.0 <= np.max(transformed) <= 2.0
2929

3030

31-
def test_heavy_tailed_distribution(encoder):
32-
"""Test that heavy-tailed distribution is correctly identified and transformed."""
33-
# Generate t-distribution data (heavy-tailed)
34-
np.random.seed(42)
35-
data = np.random.standard_t(df=3, size=(100, 1))
31+
# def test_heavy_tailed_distribution(encoder):
32+
# """Test that heavy-tailed distribution is correctly identified and transformed."""
33+
# # Generate t-distribution data (heavy-tailed)
34+
# np.random.seed(42)
35+
# data = np.random.standard_t(df=3, size=(100, 1))
3636

37-
# Force heavy-tailed distribution type
38-
encoder.prefered_distribution = DistributionType.HEAVY_TAILED
37+
# # Force heavy-tailed distribution type
38+
# encoder.prefered_distribution = DistributionType.HEAVY_TAILED
3939

40-
# Transform the data
41-
transformed = encoder(data)
40+
# # Transform the data
41+
# transformed = encoder(data)
4242

43-
# Check that the output is finite and in a reasonable range
44-
assert np.all(np.isfinite(transformed))
45-
assert 0.0 <= np.min(transformed) <= 1.0
46-
assert 0.0 <= np.max(transformed) <= 1.0
43+
# # Check that the output is finite and in a reasonable range
44+
# assert np.all(np.isfinite(transformed))
45+
# assert 0.0 <= np.min(transformed) <= 1.0
46+
# assert 0.0 <= np.max(transformed) <= 1.0
4747

4848

49-
def test_multimodal_distribution(encoder):
50-
"""Test that multimodal distribution is correctly identified and transformed."""
51-
# Generate bimodal distribution
52-
np.random.seed(42)
53-
data = np.concatenate(
54-
[np.random.normal(-3, 1, (50, 1)), np.random.normal(3, 1, (50, 1))]
55-
)
49+
# def test_multimodal_distribution(encoder):
50+
# """Test that multimodal distribution is correctly identified and transformed."""
51+
# # Generate bimodal distribution
52+
# np.random.seed(42)
53+
# data = np.concatenate(
54+
# [np.random.normal(-3, 1, (50, 1)), np.random.normal(3, 1, (50, 1))]
55+
# )
5656

57-
# Force multimodal distribution type
58-
encoder.prefered_distribution = DistributionType.MULTIMODAL
57+
# # Force multimodal distribution type
58+
# encoder.prefered_distribution = DistributionType.MULTIMODAL
5959

60-
# Transform the data
61-
transformed = encoder(data)
60+
# # Transform the data
61+
# transformed = encoder(data)
6262

63-
# Check that the output is finite and in a reasonable range
64-
assert np.all(np.isfinite(transformed))
65-
assert 0.0 <= np.min(transformed) <= 1.0
66-
assert 0.0 <= np.max(transformed) <= 1.0
63+
# # Check that the output is finite and in a reasonable range
64+
# assert np.all(np.isfinite(transformed))
65+
# assert 0.0 <= np.min(transformed) <= 1.0
66+
# assert 0.0 <= np.max(transformed) <= 1.0
6767

6868

69-
def test_uniform_distribution(encoder):
70-
"""Test that uniform distribution is correctly identified and transformed."""
71-
# Generate uniform distribution data
72-
np.random.seed(42)
73-
data = np.random.uniform(-1, 1, (100, 1))
69+
# def test_uniform_distribution(encoder):
70+
# """Test that uniform distribution is correctly identified and transformed."""
71+
# # Generate uniform distribution data
72+
# np.random.seed(42)
73+
# data = np.random.uniform(-1, 1, (100, 1))
7474

75-
# Force uniform distribution type
76-
encoder.prefered_distribution = DistributionType.UNIFORM
75+
# # Force uniform distribution type
76+
# encoder.prefered_distribution = DistributionType.UNIFORM
7777

78-
# Transform the data
79-
transformed = encoder(data)
78+
# # Transform the data
79+
# transformed = encoder(data)
8080

81-
# Check that the output is finite and in a reasonable range
82-
assert np.all(np.isfinite(transformed))
83-
assert 0.0 <= np.min(transformed) <= 1.0
84-
assert 0.0 <= np.max(transformed) <= 1.0
81+
# # Check that the output is finite and in a reasonable range
82+
# assert np.all(np.isfinite(transformed))
83+
# assert 0.0 <= np.min(transformed) <= 1.0
84+
# assert 0.0 <= np.max(transformed) <= 1.0
8585

8686

87-
def test_discrete_distribution(encoder):
88-
"""Test that discrete distribution is correctly identified and transformed."""
89-
# Generate discrete data
90-
data = np.array([[1], [2], [3], [1], [2], [3], [1], [2], [3]])
87+
# def test_discrete_distribution(encoder):
88+
# """Test that discrete distribution is correctly identified and transformed."""
89+
# # Generate discrete data
90+
# data = np.array([[1], [2], [3], [1], [2], [3], [1], [2], [3]])
9191

92-
# Force discrete distribution type
93-
encoder.prefered_distribution = DistributionType.DISCRETE
92+
# # Force discrete distribution type
93+
# encoder.prefered_distribution = DistributionType.DISCRETE
9494

95-
# Transform the data
96-
transformed = encoder(data)
95+
# # Transform the data
96+
# transformed = encoder(data)
9797

98-
# Check that the output is finite and in a reasonable range
99-
assert np.all(np.isfinite(transformed))
100-
assert 0.0 <= np.min(transformed) <= 1.0
101-
assert 0.0 <= np.max(transformed) <= 1.0
98+
# # Check that the output is finite and in a reasonable range
99+
# assert np.all(np.isfinite(transformed))
100+
# assert 0.0 <= np.min(transformed) <= 1.0
101+
# assert 0.0 <= np.max(transformed) <= 1.0
102102

103-
# Check that the discrete values are mapped to distinct values
104-
unique_values = np.unique(transformed)
105-
assert len(unique_values) == 3
103+
# # Check that the discrete values are mapped to distinct values
104+
# unique_values = np.unique(transformed)
105+
# assert len(unique_values) == 3
106106

107107

108-
def test_sparse_distribution(encoder):
109-
"""Test that sparse distribution is correctly identified and transformed."""
110-
# Generate sparse data (mostly zeros)
111-
np.random.seed(42)
112-
data = np.zeros((100, 1))
113-
data[np.random.choice(100, 10)] = np.random.exponential(1, 10)
108+
# def test_sparse_distribution(encoder):
109+
# """Test that sparse distribution is correctly identified and transformed."""
110+
# # Generate sparse data (mostly zeros)
111+
# np.random.seed(42)
112+
# data = np.zeros((100, 1))
113+
# data[np.random.choice(100, 10)] = np.random.exponential(1, 10)
114114

115-
# Force sparse distribution type
116-
encoder.prefered_distribution = DistributionType.SPARSE
115+
# # Force sparse distribution type
116+
# encoder.prefered_distribution = DistributionType.SPARSE
117117

118-
# Transform the data
119-
transformed = encoder(data)
118+
# # Transform the data
119+
# transformed = encoder(data)
120120

121-
# Check that the output is finite
122-
assert np.all(np.isfinite(transformed))
121+
# # Check that the output is finite
122+
# assert np.all(np.isfinite(transformed))
123123

124-
# Check that zeros in input remain zeros in output
125-
zero_indices = np.where(np.abs(data) < 1e-6)[0]
126-
assert np.all(np.abs(transformed[zero_indices]) < 1e-6)
124+
# # Check that zeros in input remain zeros in output
125+
# zero_indices = np.where(np.abs(data) < 1e-6)[0]
126+
# assert np.all(np.abs(transformed[zero_indices]) < 1e-6)
127127

128128

129-
def test_periodic_distribution(encoder):
130-
"""Test that periodic distribution is correctly identified and transformed."""
131-
# Generate periodic data
132-
x = np.linspace(0, 4 * np.pi, 100).reshape(-1, 1)
133-
data = np.sin(x)
129+
# def test_periodic_distribution(encoder):
130+
# """Test that periodic distribution is correctly identified and transformed."""
131+
# # Generate periodic data
132+
# x = np.linspace(0, 4 * np.pi, 100).reshape(-1, 1)
133+
# data = np.sin(x)
134134

135-
# Force periodic distribution type
136-
encoder.prefered_distribution = DistributionType.PERIODIC
135+
# # Force periodic distribution type
136+
# encoder.prefered_distribution = DistributionType.PERIODIC
137137

138-
# Transform the data
139-
transformed = encoder(data)
138+
# # Transform the data
139+
# transformed = encoder(data)
140140

141-
# Check that the output is finite
142-
assert np.all(np.isfinite(transformed))
141+
# # Check that the output is finite
142+
# assert np.all(np.isfinite(transformed))
143143

144-
# Check that the output has the expected shape (should be 2D for sine/cosine features)
145-
assert transformed.shape[1] == 2
144+
# # Check that the output has the expected shape (should be 2D for sine/cosine features)
145+
# assert transformed.shape[1] == 2
146146

147147

148-
def test_graph_mode_compatibility(encoder):
149-
"""Test that the encoder works in graph mode."""
150-
# Create a simple model with the encoder
151-
inputs = tf.keras.layers.Input(shape=(1,))
152-
encoded = encoder(inputs)
153-
outputs = tf.keras.layers.Dense(1)(encoded)
154-
model = tf.keras.Model(inputs=inputs, outputs=outputs)
148+
# def test_graph_mode_compatibility(encoder):
149+
# """Test that the encoder works in graph mode."""
150+
# # Create a simple model with the encoder
151+
# inputs = tf.keras.layers.Input(shape=(1,))
152+
# encoded = encoder(inputs)
153+
# outputs = tf.keras.layers.Dense(1)(encoded)
154+
# model = tf.keras.Model(inputs=inputs, outputs=outputs)
155155

156-
# Compile the model
157-
model.compile(optimizer="adam", loss="mse")
156+
# # Compile the model
157+
# model.compile(optimizer="adam", loss="mse")
158158

159-
# Generate some data
160-
np.random.seed(42)
161-
data = np.random.normal(0, 1, (100, 1))
162-
targets = np.random.normal(0, 1, (100, 1))
159+
# # Generate some data
160+
# np.random.seed(42)
161+
# data = np.random.normal(0, 1, (100, 1))
162+
# targets = np.random.normal(0, 1, (100, 1))
163163

164-
# Train for one step to ensure graph compatibility
165-
model.fit(data, targets, epochs=1, verbose=0)
164+
# # Train for one step to ensure graph compatibility
165+
# model.fit(data, targets, epochs=1, verbose=0)
166166

167-
# If we got here without errors, the test passes
168-
assert True
167+
# # If we got here without errors, the test passes
168+
# assert True

‎test/test_processor.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,7 @@
77
import pandas as pd
88
import tensorflow as tf
99

10-
from kdp.custom_layers import (
11-
DistributionType,
12-
)
10+
from kdp.layers.distribution_aware_encoder_layer import DistributionType
1311

1412
from kdp.layers.multi_resolution_tabular_attention_layer import (
1513
MultiResolutionTabularAttention,

0 commit comments

Comments
 (0)