It would be beneficial for performance to be able to apply TensorRT optimizations to the model.
Sadly, applying them like this
Traceback (most recent call last):
File "/home/greg/dream-inference/simple_test.py", line 33, in <module>
output = model.diffusion_generate(
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/home/greg/.cache/huggingface/modules/transformers_modules/Dream-org/Dream-v0-Instruct-7B/05334cb9faaf763692dcf9d8737c642be2b2a6ae/generation_utils.py", line 355, in diffusion_generate
result = self._sample(
^^^^^^^^^^^^^
File "/home/greg/.cache/huggingface/modules/transformers_modules/Dream-org/Dream-v0-Instruct-7B/05334cb9faaf763692dcf9d8737c642be2b2a6ae/generation_utils.py", line 411, in _sample
logits = self(x, attention_mask, tok_idx).logits
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch_tensorrt/dynamo/runtime/_MutableTorchTensorRTModule.py", line 516, in __call__
return self.forward(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch_tensorrt/dynamo/runtime/_MutableTorchTensorRTModule.py", line 479, in forward
self.compile()
File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch_tensorrt/dynamo/runtime/_MutableTorchTensorRTModule.py", line 360, in compile
self.exp_program = torch.export.export(
^^^^^^^^^^^^^^^^^^^^
File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/export/__init__.py", line 360, in export
return _export(
^^^^^^^^
File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/export/_trace.py", line 1100, in wrapper
raise e
File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/export/_trace.py", line 1066, in wrapper
ep = fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/export/exported_program.py", line 121, in wrapper
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/export/_trace.py", line 2120, in _export
ep = _export_for_training(
^^^^^^^^^^^^^^^^^^^^^
File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/export/_trace.py", line 1100, in wrapper
raise e
File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/export/_trace.py", line 1066, in wrapper
ep = fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/export/exported_program.py", line 121, in wrapper
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/export/_trace.py", line 1983, in _export_for_training
export_artifact = export_func( # type: ignore[operator]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/export/_trace.py", line 1352, in _strict_export_lower_to_aten_ir
gm_torch_level = _export_to_torch_ir(
^^^^^^^^^^^^^^^^^^^^
File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/export/_trace.py", line 740, in _export_to_torch_ir
gm_torch_level, _ = torch._dynamo.export(
^^^^^^^^^^^^^^^^^^^^^
File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/_dynamo/eval_frame.py", line 1677, in inner
result_traced = opt_f(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^
File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/_dynamo/eval_frame.py", line 659, in _fn
raise e.with_traceback(None) from None
torch._dynamo.exc.Unsupported: call_id with args (GetAttrVariable(ConstantVariable(NoneType: None), __init__),)
from user code:
File "/home/greg/.cache/huggingface/modules/transformers_modules/Dream-org/Dream-v0-Instruct-7B/05334cb9faaf763692dcf9d8737c642be2b2a6ae/modeling_dream.py", line 794, in forward
outputs = self.model(
File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
return forward_call(*args, **kwargs)
File "/home/greg/.cache/huggingface/modules/transformers_modules/Dream-org/Dream-v0-Instruct-7B/05334cb9faaf763692dcf9d8737c642be2b2a6ae/modeling_dream.py", line 668, in forward
past_key_values = DynamicCache()
File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/torch/_dynamo/polyfills/__init__.py", line 156, in instantiate_user_defined_class_object
obj.__init__(*args, **kwargs)
File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/transformers/cache_utils.py", line 1300, in __init__
super().__init__(layer_classes=DynamicLayer, *args, **kwargs)
File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/transformers/cache_utils.py", line 1080, in __init__
processor_kwargs, kwargs = parse_processor_args(processor_class, kwargs)
File "/home/greg/dream-inference/.venv/lib/python3.12/site-packages/transformers/cache_utils.py", line 1876, in parse_processor_args
params = list(inspect.signature(processor_class.__init__).parameters)[2:]
File "/home/greg/.local/share/uv/python/cpython-3.12.11-linux-x86_64-gnu/lib/python3.12/inspect.py", line 3348, in signature
return Signature.from_callable(obj, follow_wrapped=follow_wrapped,
File "/home/greg/.local/share/uv/python/cpython-3.12.11-linux-x86_64-gnu/lib/python3.12/inspect.py", line 3085, in from_callable
return _signature_from_callable(obj, sigcls=cls,
File "/home/greg/.local/share/uv/python/cpython-3.12.11-linux-x86_64-gnu/lib/python3.12/inspect.py", line 2538, in _signature_from_callable
obj = unwrap(obj, stop=(lambda f: hasattr(f, "__signature__")
File "/home/greg/.local/share/uv/python/cpython-3.12.11-linux-x86_64-gnu/lib/python3.12/inspect.py", line 773, in unwrap
memo = {id(f): f}
It would be beneficial for performance to be able to apply TensorRT optimizations to the model.
Sadly, applying them like this
I get