Thanks to visit codestin.com
Credit goes to github.com

Skip to content

feat: read_gbq creates order deterministically without table copy #191

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 14 commits into from
Nov 11, 2023
Merged

feat: read_gbq creates order deterministically without table copy #191

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
0e16921
feat: read_gbq creates order deterministically without table copy
TrevorBergeron Nov 9, 2023
30f7cd1
add to_json_string ibis op as monkey patch
TrevorBergeron Nov 10, 2023
16e8c6e
explicitly enforce region in read_gbq in session
TrevorBergeron Nov 10, 2023
e863bfa
Merge remote-tracking branch 'github/main' into hashorder
TrevorBergeron Nov 10, 2023
53cee99
order some ml outputs, fix test failulres
TrevorBergeron Nov 10, 2023
e614482
more test, mypy fixes
TrevorBergeron Nov 10, 2023
945742f
revert removal of session table from read_pandas
TrevorBergeron Nov 10, 2023
d8e99f6
address pr comments
TrevorBergeron Nov 10, 2023
69b8e28
Update bigframes/session/__init__.py
TrevorBergeron Nov 10, 2023
aa504a8
Merge remote-tracking branch 'github/main' into hashorder
TrevorBergeron Nov 10, 2023
85a54f5
remove outdated comment
TrevorBergeron Nov 10, 2023
ef3dd3d
make _cached return self and fix golden sql ml tests
TrevorBergeron Nov 10, 2023
2865788
cache before creating time series model to avoid snapshot
TrevorBergeron Nov 10, 2023
ee1f3c5
remove doctest lines dependent on unspecified ordering
TrevorBergeron Nov 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion bigframes/dataframe.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2719,7 +2719,8 @@ def _get_block(self) -> blocks.Block:
return self._block

def _cached(self) -> DataFrame:
return DataFrame(self._block.cached())
self._set_block(self._block.cached())
return self
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wunderbar! This should help with taking better advantage of our cached data in the cases where we do call this automatically.


_DataFrameOrSeries = typing.TypeVar("_DataFrameOrSeries")

Expand Down
20 changes: 14 additions & 6 deletions bigframes/ml/core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ def generate_text_embedding(

def forecast(self) -> bpd.DataFrame:
sql = self._model_manipulation_sql_generator.ml_forecast()
return self._session.read_gbq(sql)
return self._session.read_gbq(sql, index_col="forecast_timestamp").reset_index()

def evaluate(self, input_data: Optional[bpd.DataFrame] = None):
# TODO: validate input data schema
Expand All @@ -139,14 +139,18 @@ def centroids(self) -> bpd.DataFrame:

sql = self._model_manipulation_sql_generator.ml_centroids()

return self._session.read_gbq(sql)
return self._session.read_gbq(
sql, index_col=["centroid_id", "feature"]
).reset_index()

def principal_components(self) -> bpd.DataFrame:
assert self._model.model_type == "PCA"

sql = self._model_manipulation_sql_generator.ml_principal_components()

return self._session.read_gbq(sql)
return self._session.read_gbq(
sql, index_col=["principal_component_id", "feature"]
).reset_index()

def principal_component_info(self) -> bpd.DataFrame:
assert self._model.model_type == "PCA"
Expand Down Expand Up @@ -228,10 +232,12 @@ def create_model(
Returns: a BqmlModel, wrapping a trained model in BigQuery
"""
options = dict(options)
# Cache dataframes to make sure base table is not a snapshot
# cached dataframe creates a full copy, never uses snapshot
if y_train is None:
input_data = X_train
input_data = X_train._cached()
else:
input_data = X_train.join(y_train, how="outer")
input_data = X_train._cached().join(y_train._cached(), how="outer")
options.update({"INPUT_LABEL_COLS": y_train.columns.tolist()})

session = X_train._session
Expand Down Expand Up @@ -259,7 +265,9 @@ def create_time_series_model(
), "Time stamp data input must only contain 1 column."

options = dict(options)
input_data = X_train.join(y_train, how="outer")
# Cache dataframes to make sure base table is not a snapshot
# cached dataframe creates a full copy, never uses snapshot
input_data = X_train._cached().join(y_train._cached(), how="outer")
options.update({"TIME_SERIES_TIMESTAMP_COL": X_train.columns.tolist()[0]})
options.update({"TIME_SERIES_DATA_COL": y_train.columns.tolist()[0]})

Expand Down
3 changes: 2 additions & 1 deletion bigframes/series.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1503,7 +1503,8 @@ def _slice(
)

def _cached(self) -> Series:
return Series(self._block.cached())
self._set_block(self._block.cached())
return self


def _is_list_like(obj: typing.Any) -> typing_extensions.TypeGuard[typing.Sequence]:
Expand Down
Loading