From ceea15459361c8606a00a84141ad58246064fd20 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Fri, 12 Sep 2025 14:11:41 -0700 Subject: [PATCH 1/2] Move capnp module from internal to workerd --- src/workerd/api/capnp.c++ | 739 +++++++++++++++++++++++++++++ src/workerd/api/capnp.h | 408 ++++++++++++++++ src/workerd/io/worker-modules.h | 72 ++- src/workerd/server/workerd-api.c++ | 34 +- 4 files changed, 1251 insertions(+), 2 deletions(-) create mode 100644 src/workerd/api/capnp.c++ create mode 100644 src/workerd/api/capnp.h diff --git a/src/workerd/api/capnp.c++ b/src/workerd/api/capnp.c++ new file mode 100644 index 00000000000..c8a203d0667 --- /dev/null +++ b/src/workerd/api/capnp.c++ @@ -0,0 +1,739 @@ +#include "capnp.h" + +namespace workerd::api { + +// ======================================================================================= +// Some code here is derived from node-capnp. +// Copyright (c) 2014-2021 Kenton Varda, Sandstorm Development Group, Inc., and contributors +// Licensed under the MIT License + +#define STACK_STR(js, name, handle, sizeHint) \ + /* Read a JavaScript string, allocating it on the stack if it's small enough. */ \ + char name##_buf[sizeHint]{}; \ + kj::Array name##_heap; \ + kj::StringPtr name; \ + { \ + v8::Local v8str = jsg::check(handle->ToString(js.v8Context())); \ + char* ptr; \ + size_t len = v8str->Utf8LengthV2(js.v8Isolate); \ + if (len < sizeHint) { \ + ptr = name##_buf; \ + } else { \ + name##_heap = kj::heapArray(len + 1); \ + ptr = name##_heap.begin(); \ + } \ + v8str->WriteUtf8V2(js.v8Isolate, ptr, len); \ + name = kj::StringPtr(ptr, len); \ + } + +// Convert JS values to/from capnp. +struct JsCapnpConverter { + kj::Maybe wrapper; + + capnp::Orphan orphanFromJs(jsg::Lock& js, + kj::Maybe field, + capnp::Orphanage orphanage, + capnp::Type type, + v8::Local jsValue) { + return js.withinHandleScope([&]() -> capnp::Orphan { + switch (type.which()) { + case capnp::schema::Type::VOID: + if (jsValue->IsNull()) { + return capnp::VOID; + } + break; + case capnp::schema::Type::BOOL: + return jsValue->BooleanValue(js.v8Isolate); + case capnp::schema::Type::INT8: + return jsg::check(jsValue->Int32Value(js.v8Context())); + case capnp::schema::Type::INT16: + return jsg::check(jsValue->Int32Value(js.v8Context())); + case capnp::schema::Type::INT32: + return jsg::check(jsValue->Int32Value(js.v8Context())); + case capnp::schema::Type::UINT8: + return jsg::check(jsValue->Uint32Value(js.v8Context())); + case capnp::schema::Type::UINT16: + return jsg::check(jsValue->Uint32Value(js.v8Context())); + case capnp::schema::Type::UINT32: + return jsg::check(jsValue->Uint32Value(js.v8Context())); + case capnp::schema::Type::FLOAT32: + return jsg::check(jsValue->NumberValue(js.v8Context())); + case capnp::schema::Type::FLOAT64: + return jsg::check(jsValue->NumberValue(js.v8Context())); + case capnp::schema::Type::UINT64: { + if (jsValue->IsNumber()) { + // js->ToBigInt() doesn't work with Numbers. V8 bug? + double value = jsg::check(jsValue->NumberValue(js.v8Context())); + + // Casting a double to an integer when the double is out-of-range is UB. `0x1p64` is a + // C++17 hex double literal with value 2^64. We cannot use UINT64_MAX here because it is + // not exactly representable as a double, so casting it to double will actually change + // the value (rounding it up to 2^64). The compiler will rightly produce a warning about + // this. + if (value >= 0 && value < 0x1p64 && value == uint64_t(value)) { + return uint64_t(value); + } + } else { + // Let V8 decide what types can be implicitly cast to BigInt. + auto bi = jsg::check(jsValue->ToBigInt(js.v8Context())); + bool lossless; + uint64_t value = bi->Uint64Value(&lossless); + if (lossless) { + return value; + } + } + break; + } + case capnp::schema::Type::INT64: { + // (See comments above for UInt64 case.) + if (jsValue->IsNumber()) { + double value = jsg::check(jsValue->NumberValue(js.v8Context())); + if (value >= -0x1p63 && value < 0x1p63 && value == uint64_t(value)) { + return uint64_t(value); + } + } else { + auto bi = jsg::check(jsValue->ToBigInt(js.v8Context())); + bool lossless; + int64_t value = bi->Int64Value(&lossless); + if (lossless) { + return value; + } + } + break; + } + case capnp::schema::Type::TEXT: { + auto str = jsg::check(jsValue->ToString(js.v8Context())); + capnp::Orphan orphan = + orphanage.newOrphan(str->Utf8LengthV2(js.v8Isolate)); + str->WriteUtf8V2(js.v8Isolate, orphan.get().begin(), orphan.get().size()); + return kj::mv(orphan); + } + case capnp::schema::Type::DATA: + if (jsValue->IsArrayBuffer()) { + auto backing = jsValue.As()->GetBackingStore(); + return orphanage.newOrphanCopy(capnp::Data::Reader(kj::arrayPtr( + reinterpret_cast(backing->Data()), backing->ByteLength()))); + } else if (jsValue->IsArrayBufferView()) { + auto arrayBufferView = jsValue.As(); + auto backing = arrayBufferView->Buffer()->GetBackingStore(); + kj::ArrayPtr buffer(static_cast(backing->Data()), backing->ByteLength()); + auto sliceStart = arrayBufferView->ByteOffset(); + auto sliceEnd = sliceStart + arrayBufferView->ByteLength(); + KJ_ASSERT(buffer.size() >= sliceEnd); + return orphanage.newOrphanCopy(capnp::Data::Reader(buffer.slice(sliceStart, sliceEnd))); + } + break; + case capnp::schema::Type::LIST: { + if (jsValue->IsArray()) { + auto jsArray = jsValue.As(); + auto schema = type.asList(); + auto elementType = schema.getElementType(); + auto orphan = orphanage.newOrphan(schema, jsArray->Length()); + auto builder = orphan.get(); + if (elementType.isStruct()) { + // Struct lists can't adopt. + bool error = false; + for (uint i: kj::indices(builder)) { + auto element = jsg::check(jsArray->Get(js.v8Context(), i)); + if (element->IsObject()) { + structFromJs(js, builder[i].as(), element.As()); + } else { + error = true; + break; + } + } + if (error) break; + } else { + bool isPointerList = + builder.as().getElementSize() == capnp::ElementSize::POINTER; + for (uint i: kj::indices(builder)) { + auto jsElement = jsg::check(jsArray->Get(js.v8Context(), i)); + if (isPointerList && (jsElement->IsNull() || jsElement->IsUndefined())) { + // Skip null element. + } else { + builder.adopt(i, orphanFromJs(js, field, orphanage, elementType, jsElement)); + } + } + } + return kj::mv(orphan); + } + break; + } + case capnp::schema::Type::ENUM: { + auto schema = type.asEnum(); + if (jsValue->IsUint32()) { + return capnp::DynamicEnum(schema, jsg::check(jsValue->Uint32Value(js.v8Context()))); + } + + STACK_STR(js, name, jsValue, 32); + KJ_IF_SOME(enumerant, schema.findEnumerantByName(name)) { + return capnp::DynamicEnum(enumerant); + } + break; + } + case capnp::schema::Type::STRUCT: { + if (jsValue->IsObject()) { + auto schema = type.asStruct(); + auto orphan = orphanage.newOrphan(schema); + structFromJs(js, orphan.get(), jsValue.As()); + return kj::mv(orphan); + } + break; + } + case capnp::schema::Type::INTERFACE: { + KJ_IF_SOME(wrapper, this->wrapper) { + auto schema = type.asInterface(); + if (jsValue->IsNull()) { + auto cap = + capnp::Capability::Client(nullptr).castAs(schema); + return orphanage.newOrphanCopy(cap); + } else KJ_IF_SOME(cap, wrapper.tryUnwrapCap(js, js.v8Context(), jsValue)) { + // We were given a capability type obtained from elsewhere. + if (cap.getSchema().extends(schema)) { + return orphanage.newOrphanCopy(cap); + } + } else if (jsValue->IsObject()) { + // We were given a raw object, which we will treat as a server implementation. + auto cap = IoContext::current().getLocalCapSet().add( + kj::heap(js, schema, js.v8Ref(jsValue.As()), wrapper)); + return orphanage.newOrphanCopy(kj::mv(cap)); + } + } + break; + } + case capnp::schema::Type::ANY_POINTER: + // TODO(someday): Support this somehow? + break; + } + + KJ_IF_SOME(ff, field) { + JSG_FAIL_REQUIRE( + TypeError, "Incorrect type for Cap'n Proto field: ", ff.getProto().getName()); + } else { + JSG_FAIL_REQUIRE(TypeError, "Incorrect type for Cap'n Proto value."); + } + }); + } + + void fieldFromJs(jsg::Lock& js, + capnp::DynamicStruct::Builder builder, + capnp::StructSchema::Field field, + v8::Local jsValue) { + if (jsValue->IsUndefined()) { + // Ignore. + return; + } + auto proto = field.getProto(); + switch (proto.which()) { + case capnp::schema::Field::SLOT: { + builder.adopt(field, + orphanFromJs(js, field, capnp::Orphanage::getForMessageContaining(builder), + field.getType(), jsValue)); + return; + } + + case capnp::schema::Field::GROUP: + if (jsValue->IsObject()) { + structFromJs( + js, builder.init(field).as(), jsValue.As()); + } else { + JSG_FAIL_REQUIRE(TypeError, "Incorrect type for Cap'n Proto field: ", proto.getName()); + } + return; + } + + KJ_FAIL_ASSERT("Unimplemented field type (not slot or group)."); + } + + void structFromJs( + jsg::Lock& js, capnp::DynamicStruct::Builder builder, v8::Local jsValue) { + js.withinHandleScope([&] { + auto schema = builder.getSchema(); + v8::Local fieldNames = jsg::check(jsValue->GetOwnPropertyNames(js.v8Context())); + for (uint i: kj::zeroTo(fieldNames->Length())) { + auto jsName = jsg::check(fieldNames->Get(js.v8Context(), i)); + STACK_STR(js, fieldName, jsName, 32); + KJ_IF_SOME(field, schema.findFieldByName(fieldName)) { + fieldFromJs(js, builder, field, jsg::check(jsValue->Get(js.v8Context(), jsName))); + } else { + JSG_FAIL_REQUIRE(TypeError, "No such field in Cap'n Proto struct: ", fieldName); + } + } + }); + } + + void rpcResultsFromJs(jsg::Lock& js, + capnp::CallContext& rpcContext, + v8::Local jsValue) { + if (jsValue->IsObject()) { + structFromJs(js, rpcContext.getResults(), jsValue.As()); + } else if (jsValue->IsUndefined()) { + // assume default return + } else { + JSG_FAIL_REQUIRE(TypeError, "RPC method server implementation returned a non-object."); + } + } + + // --------------------------------------------------------------------------- + // handle pipelines (as in promise pipelining) + // + // In C++, a capnp::RemotePromise represents a combination of a Promise and a + // T::Pipeline. The latter is a special object that allows immediately initiating pipeline calls + // on any capabilities that the response is expected to contain. + // + // In JavaScript, we will accomplish something similar by returning a Promise that has been + // extended with properties representing the pipelined capabilities. + + struct PipelinedCap; + typedef kj::HashMap PipelinedCapMap; + + // We return a set of pipelined capabilities on the Promise returned by an RPC call. Later on, + // that Promise resolves to a response object likely containing the same capabilities again. + // We don't want the application to have to call `.close()` on both the pipelined version and + // the final version in order to actually close a capability. So, we need to make sure the final + // response uses the same CapnpCapability objects that were returned as part of the pipeline. + // To facilitate this, when we extend the Promise with pipeline properties, we also return a + // PipelineCapMap which contains all the objects that need to be injected into the final + // response. + struct PipelinedCap { + kj::OneOf, PipelinedCapMap> content; + }; + + v8::Local pipelineStructFieldToJs(jsg::Lock& js, + capnp::DynamicStruct::Pipeline& pipeline, + capnp::StructSchema::Field field, + PipelinedCapMap& capMap) { + v8::Local fieldValue = v8::Object::New(js.v8Isolate); + auto subMap = + pipelineToJs(js, pipeline.get(field).releaseAs(), fieldValue); + if (subMap.size() > 0) { + // Some capabilities were found in this sub-message, so add it to the map. + capMap.insert(field, PipelinedCap{kj::mv(subMap)}); + } + return fieldValue; + } + + // This function is only useful in the context of RPC, where this->wrapper will always be + // available. + PipelinedCapMap pipelineToJs( + jsg::Lock& js, capnp::DynamicStruct::Pipeline&& pipeline, v8::Local jsValue) { + CapnpTypeWrapperBase& wrapper = KJ_REQUIRE_NONNULL(this->wrapper); + + return js.withinHandleScope([&]() -> PipelinedCapMap { + capnp::StructSchema schema = pipeline.getSchema(); + + PipelinedCapMap capMap; + + for (capnp::StructSchema::Field field: schema.getNonUnionFields()) { + auto proto = field.getProto(); + v8::Local fieldValue; + + switch (proto.which()) { + case capnp::schema::Field::SLOT: { + auto type = field.getType(); + switch (type.which()) { + case capnp::schema::Type::STRUCT: + fieldValue = pipelineStructFieldToJs(js, pipeline, field, capMap); + break; + case capnp::schema::Type::ANY_POINTER: + if (type.whichAnyPointerKind() != + capnp::schema::Type::AnyPointer::Unconstrained::CAPABILITY) { + continue; + } + [[fallthrough]]; + case capnp::schema::Type::INTERFACE: { + jsg::Ref ref = nullptr; + fieldValue = wrapper.wrapCap(js, js.v8Context(), + pipeline.get(field).releaseAs(), &ref); + capMap.insert(field, PipelinedCap{kj::mv(ref)}); + break; + } + default: + continue; + } + break; + } + + case capnp::schema::Field::GROUP: + fieldValue = pipelineStructFieldToJs(js, pipeline, field, capMap); + break; + + default: + continue; + } + + KJ_ASSERT(!fieldValue.IsEmpty()); + jsg::check(jsValue->Set( + js.v8Context(), jsg::v8StrIntern(js.v8Isolate, proto.getName()), fieldValue)); + } + + return capMap; + }); + } + + // --------------------------------------------------------------------------- + // convert capnp values to JS + + v8::Local valueToJs(jsg::Lock& js, + capnp::DynamicValue::Reader value, + capnp::Type type, + kj::Maybe pipelinedCap) { + // TODO(later): support deserialization outside of RPC, i.e., not requiring a wrapper. + CapnpTypeWrapperBase& wrapper = KJ_REQUIRE_NONNULL(this->wrapper); + + return js.withinHandleScope([&]() -> v8::Local { + switch (value.getType()) { + case capnp::DynamicValue::UNKNOWN: + return js.v8Undefined(); + case capnp::DynamicValue::VOID: + return js.v8Null(); + case capnp::DynamicValue::BOOL: + return v8::Boolean::New(js.v8Isolate, value.as()); + case capnp::DynamicValue::INT: { + if (type.which() == capnp::schema::Type::INT64 || + type.which() == capnp::schema::Type::UINT64) { + return v8::BigInt::New(js.v8Isolate, value.as()); + } else { + return v8::Integer::New(js.v8Isolate, value.as()); + } + } + case capnp::DynamicValue::UINT: { + if (type.which() == capnp::schema::Type::INT64 || + type.which() == capnp::schema::Type::UINT64) { + return v8::BigInt::NewFromUnsigned(js.v8Isolate, value.as()); + } else { + return v8::Integer::NewFromUnsigned(js.v8Isolate, value.as()); + } + } + case capnp::DynamicValue::FLOAT: + return v8::Number::New(js.v8Isolate, value.as()); + case capnp::DynamicValue::TEXT: + return jsg::v8Str(js.v8Isolate, value.as()); + case capnp::DynamicValue::DATA: { + capnp::Data::Reader data = value.as(); + + // In theory we could avoid a copy if we kept the response message in memory, but we + // probably don't want to do that. + auto result = jsg::check(v8::ArrayBuffer::MaybeNew(js.v8Isolate, data.size())); + memcpy(result->GetBackingStore()->Data(), data.begin(), data.size()); + + return result; + } + case capnp::DynamicValue::LIST: { + capnp::DynamicList::Reader list = value.as(); + auto elementType = list.getSchema().getElementType(); + auto indices = kj::indices(list); + KJ_STACK_ARRAY(v8::Local, items, indices.size(), 100, 100); + for (uint i: indices) { + items[i] = valueToJs(js, list[i], elementType, kj::none); + } + return v8::Array::New(js.v8Isolate, items.begin(), items.size()); + } + case capnp::DynamicValue::ENUM: { + auto enumValue = value.as(); + KJ_IF_SOME(enumerant, enumValue.getEnumerant()) { + return jsg::v8StrIntern(js.v8Isolate, enumerant.getProto().getName()); + } else { + return v8::Integer::NewFromUnsigned(js.v8Isolate, enumValue.getRaw()); + } + } + case capnp::DynamicValue::STRUCT: { + auto capMap = pipelinedCap.map([](PipelinedCap& pc) -> PipelinedCapMap& { + // If we had a PipelinedCap for a struct field, it must be a PipelinedCapMap. + return pc.content.get(); + }); + + capnp::DynamicStruct::Reader reader = value.as(); + auto object = v8::Object::New(js.v8Isolate); + KJ_IF_SOME(field, reader.which()) { + fieldToJs(js, object, reader, field, capMap); + } + + for (auto field: reader.getSchema().getNonUnionFields()) { + if (reader.has(field)) { + fieldToJs(js, object, reader, field, capMap); + } + } + return object; + } + case capnp::DynamicValue::CAPABILITY: + KJ_IF_SOME(p, pipelinedCap) { + // Use the same CapnpCapability object that we returned earlier for promise pipelining. + // Note: We know the JS wrapper exists because CapnpCapability objects are always created + // by CapnpTypeWrapper::wrap() and immediately have a wrapper added. + return KJ_ASSERT_NONNULL(p.content.get>().tryGetHandle(js)); + } else { + return wrapper.wrapCap(js, js.v8Context(), value.as()); + } + case capnp::DynamicValue::ANY_POINTER: + return js.v8Null(); + } + + KJ_FAIL_ASSERT("Unimplemented DynamicValue type."); + }); + } + + void fieldToJs(jsg::Lock& js, + v8::Local object, + capnp::DynamicStruct::Reader reader, + capnp::StructSchema::Field field, + kj::Maybe capMap) { + js.withinHandleScope([&] { + kj::Maybe pipelinedCap; + KJ_IF_SOME(m, capMap) { + pipelinedCap = m.find(field); + } + + auto proto = field.getProto(); + v8::Local fieldValue; + switch (proto.which()) { + case capnp::schema::Field::SLOT: + fieldValue = valueToJs(js, reader.get(field), field.getType(), pipelinedCap); + break; + case capnp::schema::Field::GROUP: + fieldValue = valueToJs(js, reader.get(field), field.getType(), pipelinedCap); + break; + } + + JSG_REQUIRE( + !fieldValue.IsEmpty(), TypeError, "Unimplemented field type (not slot or group)."); + + jsg::check( + object->Set(js.v8Context(), jsg::v8StrIntern(js.v8Isolate, proto.getName()), fieldValue)); + }); + } +}; + +// ======================================================================================= + +void fillCapnpFieldFromJs(jsg::Lock& js, + capnp::DynamicStruct::Builder builder, + capnp::StructSchema::Field field, + v8::Local jsValue) { + JsCapnpConverter converter; + converter.fieldFromJs(js, builder, field, jsValue); +} + +capnp::Orphan capnpValueFromJs( + jsg::Lock& js, capnp::Orphanage orphanage, capnp::Type type, v8::Local jsValue) { + JsCapnpConverter converter; + return converter.orphanFromJs(js, kj::none, orphanage, type, jsValue); +} + +// ======================================================================================= + +CapnpServer::CapnpServer(jsg::Lock& js, + capnp::InterfaceSchema schema, + jsg::V8Ref objectParam, + CapnpTypeWrapperBase& wrapper) + : capnp::DynamicCapability::Server(schema), + ioContext(IoContext::current().getWeakRef()), + object(kj::mv(objectParam)), + closeMethod(getCloseMethod(js)), + wrapper(wrapper) {} + +kj::Maybe> CapnpServer::getCloseMethod(jsg::Lock& js) { + auto handle = object.getHandle(js); + auto methodHandle = + jsg::check(handle->Get(js.v8Context(), jsg::v8StrIntern(js.v8Isolate, "close"))); + if (methodHandle->IsFunction()) { + return js.v8Ref(methodHandle.As()); + } else { + return kj::none; + } +} + +CapnpServer::~CapnpServer() noexcept(false) { + KJ_IF_SOME(c, closeMethod) { + ioContext->runIfAlive([&](IoContext& rc) { + rc.addTask( + rc.run([object = kj::mv(object), closeMethod = kj::mv(c)](Worker::Lock& lock) mutable { + auto handle = object.getHandle(lock); + auto methodHandle = closeMethod.getHandle(lock); + if (methodHandle->IsFunction()) { + jsg::check(methodHandle.As()->Call(lock.getContext(), handle, 0, nullptr)); + } + })); + }); + } +} + +kj::Promise CapnpServer::call(capnp::InterfaceSchema::Method method, + capnp::CallContext rpcContext) { + kj::Promise result = nullptr; + + bool live = ioContext->runIfAlive([&](IoContext& rc) { + result = + rc.run([this, method, rpcContext, &rc](Worker::Lock& lock) mutable -> kj::Promise { + jsg::Lock& js = lock; + auto handle = object.getHandle(js); + auto methodName = method.getProto().getName(); + auto methodHandle = + jsg::check(handle->Get(lock.getContext(), jsg::v8StrIntern(js.v8Isolate, methodName))); + + if (!methodHandle->IsFunction()) { + KJ_UNIMPLEMENTED(kj::str("jsg.Error: RPC method not implemented: ", methodName)); + } + + JsCapnpConverter converter{wrapper}; + auto params = rpcContext.getParams(); + auto jsParams = converter.valueToJs(js, params, params.getSchema(), kj::none); + rpcContext.releaseParams(); + + auto result = jsg::check( + methodHandle.As()->Call(lock.getContext(), handle, 1, &jsParams)); + KJ_IF_SOME(promise, wrapper.tryUnwrapPromise(lock, lock.getContext(), result)) { + return rc.awaitJs(js, + promise.then( + js, rc.addFunctor([this, rpcContext](jsg::Lock& js, jsg::Value result) mutable { + JsCapnpConverter converter{wrapper}; + converter.rpcResultsFromJs(js, rpcContext, result.getHandle(js)); + }))); + + } else { + converter.rpcResultsFromJs(js, rpcContext, result); + return kj::READY_NOW; + } + }); + }); + + if (live) { + return result; + } else { + return KJ_EXCEPTION(DISCONNECTED, "jsg.Error: Called to event context that is no longer live."); + } +} + +// ======================================================================================= + +CapnpCapability::CapnpCapability(capnp::DynamicCapability::Client client) + : schema(client.getSchema()), + client(IoContext::current().addObject(kj::heap(kj::mv(client)))) {} + +CapnpCapability::~CapnpCapability() noexcept(false) { + KJ_IF_SOME(c, client) { + // The client was not explicitly close()ed and instead waited for GC. There are two problems + // with this: + // 1. It's rude to force the remote peer to wait until the lazy garbage collector gets around + // to collecting the object before we let the peer know that it can clean up its end. Our + // GC is sociopathic, it decides when to collect based purely on its own memory pressure + // and has no idea what memory pressure the peer might be feeling, so likely won't make + // empathetic choices about when to collect. + // 2. We generally do not want to allow an application to observe its own garbage collection + // behavior, as this may reveal side channels. The capability could be a loopback into + // this very isolate, in which case closing it now would immediately call back into the + // server's close() method, notifying the application of its own GC. We need to prevent that. + + // To solve #2, we defer destruction of the object until the end of the IoContext. + kj::mv(c).deferGcToContext(); + + // In preview, let's try to warn the developer about the problem. + // + // TODO(cleanup): Instead of logging this warning at GC time, it would be better if we logged + // it at the time that the client is destroyed, i.e. when the IoContext is torn down, + // which is usually sooner (and more deterministic). But logging a warning during + // IoContext tear-down is problematic since logWarningOnce() is a method on + // IoContext... + if (IoContext::hasCurrent()) { + IoContext::current().logWarningOnce( + kj::str("A Cap'n Proto capability of type ", schema.getShortDisplayName(), + " was not closed properly. You must call close() on all capabilities in order to " + "let the other side know that you are no longer using them. You cannot rely on " + "the garbage collector for this because it may take arbitrarily long before actually " + "collecting unreachable objects.")); + } + } +} + +v8::Local CapnpCapability::call(jsg::Lock& js, + capnp::InterfaceSchema::Method method, + v8::Local params, + CapnpTypeWrapperBase& wrapper) { + auto& ioContext = IoContext::current(); + auto req = getClient(js, wrapper).newRequest(method); + JsCapnpConverter converter{wrapper}; + if (params->IsObject()) { + converter.structFromJs(js, req, params.As()); + } else if (params->IsUndefined()) { + // leave params all-default + } else { + JSG_FAIL_REQUIRE(TypeError, "Argument to a capnp RPC call must be an object."); + } + if (method.isStreaming()) { + // Note: We know the JS wrapper exists for JSG_THIS because CapnpCapability objects are always + // created by CapnpTypeWrapper::wrap() and immediately have a wrapper added. + return wrapper.wrapPromise(js, js.v8Context(), KJ_ASSERT_NONNULL(JSG_THIS.tryGetHandle(js)), + ioContext.awaitIo( + js, req.sendStreaming(), [](jsg::Lock& js) { return js.v8Ref(js.v8Undefined()); })); + } else { + // The RPC promise is actually both a promise and a pipeline. + auto rpcPromise = req.send(); + + auto pipelinedCapHolder = kj::heap(); + auto& pipelinedCapRef = *pipelinedCapHolder; + + // We'll consume the promise itself to handle converting the response. + // Note: We know the JS wrapper exists for JSG_THIS because CapnpCapability objects are always + // created by CapnpTypeWrapper::wrap() and immediately have a wrapper added. + auto responsePromise = + kj::Promise>(kj::mv(rpcPromise)) + .catch_([](kj::Exception&& ex) -> kj::Promise> { + auto errorType = jsg::tunneledErrorType(ex.getDescription()); + if (!errorType.isJsgError) { + // Wrap any non-JS exceptions as JS errors + auto newDescription = + kj::str("remote." JSG_EXCEPTION(Error) ": capnp RPC exception: "_kj, errorType.message); + ex.setDescription(kj::mv(newDescription)); + } + return kj::mv(ex); + }); + auto result = + wrapper.wrapPromise(js, js.v8Context(), KJ_ASSERT_NONNULL(JSG_THIS.tryGetHandle(js)), + ioContext.awaitIo(js, kj::mv(responsePromise), + [&wrapper, pipelinedCapHolder = kj::mv(pipelinedCapHolder)]( + jsg::Lock& js, capnp::Response resp) mutable { + JsCapnpConverter converter{wrapper}; + return js.v8Ref(converter.valueToJs(js, resp, resp.getSchema(), *pipelinedCapHolder)); + })); + + // Now we take the pipeline part of `rpcPromise` and merge it into the V8 promise object, by + // adding fields representing the pipelined struct. + KJ_ASSERT(result->IsPromise()); + pipelinedCapRef.content = + converter.pipelineToJs(js, kj::mv(rpcPromise), result.As()); + + return result; + } +} + +void CapnpCapability::close() { + KJ_IF_SOME(c, client) { + // Verify we're in the correct IoContext. This will throw otherwise. + *c; + } + client = kj::none; +} + +jsg::Promise>> CapnpCapability::unwrap(jsg::Lock& js) { + // We need to allocate a heap copy of the `Client` so that if this capability is closed while + // the promise is still outstanding, the client isn't destroyed, which would otherwise cause + // UAF in the getLocalServer() implementation. + auto capHolder = kj::heap(*JSG_REQUIRE_NONNULL(client, Error, "Capability has been closed.")); + auto& ioContext = IoContext::current(); + auto promise = ioContext.getLocalCapSet().getLocalServer(*capHolder); + + return ioContext.awaitIo(js, kj::mv(promise), + [capHolder = kj::mv(capHolder)]( + jsg::Lock& js, kj::Maybe server) { + return server.map([&](capnp::DynamicCapability::Server& s) { + return kj::downcast(s).object.addRef(js); + }); + }); +} + +capnp::DynamicCapability::Client CapnpCapability::getClient( + jsg::Lock&, CapnpTypeWrapperBase& wrapper) { + return *JSG_REQUIRE_NONNULL(client, Error, "Capability has been closed."); +} + +} // namespace workerd::api diff --git a/src/workerd/api/capnp.h b/src/workerd/api/capnp.h new file mode 100644 index 00000000000..517fbce81bc --- /dev/null +++ b/src/workerd/api/capnp.h @@ -0,0 +1,408 @@ +#pragma once + +#include +#include + +#include +#include + +namespace workerd::api { + +template +class CapnpTypeWrapper; +class CapnpCapability; +class CapnpTypeWrapperBase; + +void fillCapnpFieldFromJs(capnp::DynamicStruct::Builder builder, + capnp::StructSchema::Field field, + v8::Local context, + v8::Local jsValue); + +capnp::Orphan capnpValueFromJs( + jsg::Lock& js, capnp::Orphanage orphanage, capnp::Type type, v8::Local jsValue); + +class CapnpServer final: public capnp::DynamicCapability::Server { + public: + CapnpServer(jsg::Lock& js, + capnp::InterfaceSchema schema, + jsg::V8Ref object, + CapnpTypeWrapperBase& wrapper); + ~CapnpServer() noexcept(false); + + kj::Promise call(capnp::InterfaceSchema::Method method, + capnp::CallContext context) override; + + private: + kj::Own ioContext; + jsg::V8Ref object; + kj::Maybe> closeMethod; + CapnpTypeWrapperBase& wrapper; // only valid if isolate is locked! + + kj::Maybe> getCloseMethod(jsg::Lock& js); + + friend class CapnpCapability; +}; + +class CapnpCapability: public jsg::Object { + public: + CapnpCapability(capnp::DynamicCapability::Client client); + ~CapnpCapability() noexcept(false); + + v8::Local call(jsg::Lock& js, + capnp::InterfaceSchema::Method method, + v8::Local params, + CapnpTypeWrapperBase& wrapper); + + void close(); + jsg::Promise>> unwrap(jsg::Lock& js); + + JSG_RESOURCE_TYPE(CapnpCapability) { + JSG_METHOD(close); + JSG_METHOD(unwrap); + } + + capnp::DynamicCapability::Client getClient(jsg::Lock& js, CapnpTypeWrapperBase& wrapper); + + private: + // Used for error messages. + capnp::InterfaceSchema schema; + + // null if closed + kj::Maybe> client; + + template + friend class CapnpTypeWrapper; +}; + +class CapnpTypeWrapperBase { + public: + virtual v8::Local wrapCap(jsg::Lock& js, + v8::Local context, + capnp::DynamicCapability::Client value, + kj::Maybe&> refToInitialize = kj::none) = 0; + virtual kj::Maybe tryUnwrapCap( + jsg::Lock& js, v8::Local context, v8::Local value) = 0; + + virtual v8::Local wrapPromise(jsg::Lock& js, + v8::Local context, + kj::Maybe> creator, + jsg::Promise value) = 0; + virtual kj::Maybe> tryUnwrapPromise( + jsg::Lock& js, v8::Local context, v8::Local value) = 0; +}; + +template +class CapnpTypeWrapper: private CapnpTypeWrapperBase { + public: + static constexpr const char* getName(capnp::DynamicCapability::Client*) { + return "Capability"; + } + + v8::Local wrap(jsg::Lock& js, + v8::Local context, + kj::Maybe> creator, + capnp::Schema schema) { + auto tmpl = getCapnpTemplate(js, schema); + return jsg::check(tmpl->GetFunction(context)); + } + + v8::Local wrap(jsg::Lock& js, + v8::Local context, + kj::Maybe> creator, + capnp::DynamicCapability::Client client, + kj::Maybe&> refToInitialize = kj::none) { + auto tmpl = getCapnpTemplate(js, client.getSchema()); + auto obj = jsg::check(tmpl->InstanceTemplate()->NewInstance(context)); + auto ref = js.alloc(kj::mv(client)); + ref.attachWrapper(js.v8Isolate, obj); + KJ_IF_SOME(r, refToInitialize) { + r = kj::mv(ref); + } + return obj; + } + + // Wrap a specific compiled-in interface. This lets you use MyType::Client as a return type + // in a JSG method. + template > + v8::Local wrap(jsg::Lock& js, + v8::Local context, + kj::Maybe> creator, + Client client) { + return wrap(js, context, creator, capnp::toDynamic(kj::mv(client))); + } + + kj::Maybe tryUnwrap(jsg::Lock& js, + v8::Local context, + v8::Local handle, + capnp::DynamicCapability::Client*, + kj::Maybe> parentObject) { + auto& wrapper = static_cast(*this); + KJ_IF_SOME(obj, + wrapper.tryUnwrap(js, context, handle, (CapnpCapability*)nullptr, parentObject)) { + return obj.getClient(js, *this); + } else { + // Since we don't know the schema, we cannot accept an arbitrary object. + return kj::none; + } + } + + // Unwrap a specific compiled-in interface. This lets you use MyType::Client as a parameter + // in a JSG method. + template > + kj::Maybe tryUnwrap(jsg::Lock& js, + v8::Local context, + v8::Local handle, + Client*, + kj::Maybe> parentObject) { + auto expectedSchema = capnp::Schema::from(); + + auto& wrapper = static_cast(*this); + KJ_IF_SOME(obj, + wrapper.tryUnwrap(js, context, handle, (CapnpCapability*)nullptr, parentObject)) { + capnp::DynamicCapability::Client dynamic = obj.getClient(js.v8Isolate, *this); + if (dynamic.getSchema().extends(expectedSchema)) { + return dynamic.as(); + } else { + // Incompatible interfaces. + return kj::none; + } + } else if (handle->IsObject()) { + // Treat object as a server implementation. + auto isolate = js.v8Isolate; + CapnpTypeWrapperBase& wrapper = TypeWrapper::from(isolate); + capnp::DynamicCapability::Client dynamic = IoContext::current().getLocalCapSet().add( + kj::heap(expectedSchema, handle.As(), wrapper, isolate)); + return dynamic.as(); + } else { + return kj::none; + } + } + + // Not relevant for us but we must define a method with this name to satisfy TypeWrapperExtension. + void newContext() = delete; + + template > + v8::Local getTemplate(jsg::Lock& js, Client*) { + static_assert(!isContext); + return getCapnpTemplate(js, capnp::Schema::from()); + } + + v8::Local getCapnpTemplate(jsg::Lock& js, capnp::Schema schema) { + using Ret = typename decltype(typeConstructors)::Entry; + return typeConstructors + .findOrCreate(schema, [&]() -> Ret { + return js.withinHandleScope([&]() -> Ret { + auto handle = makeConstructor(js, schema); + return {schema, {js.v8Isolate, handle}}; + }); + }).Get(js.v8Isolate); + } + + private: + kj::HashMap> typeConstructors; + + // Each method callback we create needs to pack the method schema into a v8::External. But + // v8::External can only store a pointer, and InterfaceSchema::Method is larger than a pointer. + // So we need to allocate copies of all the `Method` objects somewhere where they'll live until + // the isolate shuts down. + kj::HashMap> methodSchemas; + + v8::Local makeConstructor(jsg::Lock& js, capnp::Schema schema) { + return js.withinHandleScope([&]() -> v8::Local { + // HACK: We happen to know that `Schema` is just a pointer internally, and is + // trivially copyable and destructible. So, we can safely stuff it directly into a + // v8::External by value, avoiding extra allocations. + static_assert(sizeof(schema) == sizeof(void*)); + static_assert(__is_trivially_copyable(capnp::Schema)); + static_assert(__is_trivially_destructible(capnp::Schema)); + void* schemaAsPtr; + memcpy(&schemaAsPtr, &schema, sizeof(schema)); + + auto constructor = v8::FunctionTemplate::New( + js.v8Isolate, &constructorCallback, v8::External::New(js.v8Isolate, schemaAsPtr)); + + auto prototype = constructor->PrototypeTemplate(); + auto signature = v8::Signature::New(js.v8Isolate, constructor); + + auto instance = constructor->InstanceTemplate(); + + constructor->SetClassName(jsg::v8StrIntern(js.v8Isolate, schema.getShortDisplayName())); + + auto& wrapper = static_cast(*this); + + auto proto = schema.getProto(); + switch (proto.which()) { + case capnp::schema::Node::FILE: + case capnp::schema::Node::STRUCT: + case capnp::schema::Node::ENUM: + case capnp::schema::Node::CONST: + case capnp::schema::Node::ANNOTATION: + // TODO(someday): Support non-interface types. + break; + + case capnp::schema::Node::INTERFACE: { + // As explained in ResourceWrapper, we must have 2 internal fields, where the first one is + // the GC visitation callback. + instance->SetInternalFieldCount(jsg::Wrappable::INTERNAL_FIELD_COUNT); + + constructor->Inherit(wrapper.getTemplate(js.v8Isolate, (CapnpCapability*)nullptr)); + kj::HashSet seen; + addAllMethods(js, prototype, signature, schema.asInterface(), seen); + break; + } + } + + for (auto nested: proto.getNestedNodes()) { + KJ_IF_SOME(child, + js.getCapnpSchemaLoader().tryGet(nested.getId())) { + switch (child.getProto().which()) { + case capnp::schema::Node::FILE: + case capnp::schema::Node::STRUCT: + case capnp::schema::Node::INTERFACE: + constructor->Set( + jsg::v8StrIntern(js.v8Isolate, nested.getName()), makeConstructor(js, child)); + break; + + case capnp::schema::Node::ENUM: + case capnp::schema::Node::CONST: + case capnp::schema::Node::ANNOTATION: + // These kinds are not implemented and cannot contain further nested scopes, so don't + // generate anything at all for now. + break; + } + } + } + + return constructor; + }); + } + + // Add all methods to the capability prototype. Since JavaScript doesn't support multiple + // inheritance, we need to flatten all inherited methods into each interface. + // + // `seen` is a set of type IDs that we've visited already, so that diamond inheritance doesn't + // lead to us double-registering methods. + void addAllMethods(jsg::Lock& js, + v8::Local prototype, + v8::Local signature, + capnp::InterfaceSchema schema, + kj::HashSet& seen) { + + JSG_REQUIRE(seen.size() < 64, TypeError, + "Interface inherits too many types: ", schema.getProto().getDisplayName()); + + // Reverse-iterate so that in case of duplicate method names, the method from the first class + // in the list takes precedence. + auto supers = schema.getSuperclasses(); + for (auto i = supers.size(); i > 0; i--) { + auto super = supers[i - 1]; + + // Check if this superclass is in the `seen` set. As a slight optimization we only check this + // before visiting a superclass, so that for a regular interface that doesn't inherit + // anything, we never allocate the `seen` set. This assumes that inheritance is not cyclic. + // Technically it's possible to declare cyclic inheritance (maliciously, perhaps), but in + // that case we'll just redundantly create the methods for one type, which is not a big deal. + uint64_t id = super.getProto().getId(); + bool isNew = false; + seen.findOrCreate(id, [&]() { + isNew = true; + return id; + }); + if (isNew) { + addAllMethods(js, prototype, signature, super, seen); + } + } + + kj::ArrayPtr methods = + methodSchemas.findOrCreate(schema, [&]() -> typename decltype(methodSchemas)::Entry { + return {schema, KJ_MAP(m, schema.getMethods()) { return m; }}; + }); + + for (auto& method: methods) { + auto name = jsg::v8StrIntern(js.v8Isolate, method.getProto().getName()); + prototype->Set(name, + v8::FunctionTemplate::New(js.v8Isolate, &methodCallback, + v8::External::New(js.v8Isolate, &method), signature, 0, + v8::ConstructorBehavior::kThrow)); + } + } + + static void constructorCallback(const v8::FunctionCallbackInfo& args) { + jsg::liftKj(args, [&]() { + auto data = args.Data(); + KJ_ASSERT(data->IsExternal()); + void* schemaAsPtr = data.As()->Value(); + capnp::Schema schema; + memcpy(&schema, &schemaAsPtr, sizeof(schema)); + + JSG_REQUIRE(args.IsConstructCall(), TypeError, "Failed to construct '", + schema.getShortDisplayName(), + "': Please use the " + "'new' operator, this object constructor cannot be called as a function."); + + auto& js = jsg::Lock::from(args.GetIsolate()); + auto obj = args.This(); + KJ_ASSERT(obj->InternalFieldCount() == jsg::Wrappable::INTERNAL_FIELD_COUNT); + + auto arg = args[0]; + JSG_REQUIRE(arg->IsObject(), TypeError, "Constructor argument for '", + schema.getShortDisplayName(), + "' must be an object " + "implementing the interface."); + + CapnpTypeWrapperBase& wrapper = TypeWrapper::from(js.v8Isolate); + capnp::DynamicCapability::Client client = + IoContext::current().getLocalCapSet().add(kj::heap(js, schema.asInterface(), + jsg::V8Ref(js.v8Isolate, arg.As()), wrapper)); + auto ptr = js.alloc(kj::mv(client)); + + ptr.attachWrapper(js.v8Isolate, obj); + }); + } + + static void methodCallback(const v8::FunctionCallbackInfo& args) { + jsg::liftKj(args, [&]() { + auto data = args.Data(); + KJ_ASSERT(data->IsExternal()); + auto& method = + *reinterpret_cast(data.As()->Value()); + + auto& js = jsg::Lock::from(args.GetIsolate()); + auto obj = args.This(); + auto& wrapper = TypeWrapper::from(js.v8Isolate); + auto& self = jsg::extractInternalPointer(js.v8Context(), obj); + + return wrapper.wrap(js, js.v8Context(), obj, self.call(js, method, args[0], wrapper)); + }); + } + + v8::Local wrapCap(jsg::Lock& js, + v8::Local context, + capnp::DynamicCapability::Client value, + kj::Maybe&> refToInitialize) override { + return wrap(js, context, kj::none, kj::mv(value), refToInitialize); + } + kj::Maybe tryUnwrapCap( + jsg::Lock& js, v8::Local context, v8::Local value) override { + return tryUnwrap(js, context, value, (capnp::DynamicCapability::Client*)nullptr, kj::none); + } + + v8::Local wrapPromise(jsg::Lock& js, + v8::Local context, + kj::Maybe> creator, + jsg::Promise value) override { + return static_cast(*this).wrap(js, context, creator, kj::mv(value)); + } + kj::Maybe> tryUnwrapPromise( + jsg::Lock& js, v8::Local context, v8::Local value) override { + return static_cast(*this).tryUnwrap( + js, context, value, (jsg::Promise*)nullptr, kj::none); + } +}; + +#define EW_CAPNP_TYPES \ + ::workerd::api::CapnpCapability, \ + ::workerd::jsg::TypeWrapperExtension<::workerd::api::CapnpTypeWrapper> +// The list of capnp.h types that are added to worker.c++'s JSG_DECLARE_ISOLATE_TYPE + +} // namespace workerd::api diff --git a/src/workerd/io/worker-modules.h b/src/workerd/io/worker-modules.h index fa80fe1ba9a..dbc191b7d82 100644 --- a/src/workerd/io/worker-modules.h +++ b/src/workerd/io/worker-modules.h @@ -139,7 +139,77 @@ static kj::Arc newWorkerModuleRegistry( break; } KJ_CASE_ONEOF(content, Worker::Script::CapnpModule) { - KJ_FAIL_REQUIRE("capnp modules are not yet supported in workerd"); + auto& schemaLoader = builder.getSchemaLoader(); + auto schema = schemaLoader.get(content.typeId); + kj::Vector exports; + for (auto nested: schema.getProto().getNestedNodes()) { + auto child = schemaLoader.get(nested.getId()); + switch (child.getProto().which()) { + case capnp::schema::Node::FILE: + case capnp::schema::Node::STRUCT: + case capnp::schema::Node::INTERFACE: { + exports.add(kj::str(nested.getName())); + break; + } + case capnp::schema::Node::ENUM: + case capnp::schema::Node::CONST: + case capnp::schema::Node::ANNOTATION: + // These kinds are not implemented and cannot contain further nested scopes, so + // don't generate anything at all for now. + break; + } + } + + bundleBuilder.addSyntheticModule(def.name, + [typeId = content.typeId](jsg::Lock& js, const jsg::Url&, + const jsg::modules::Module::ModuleNamespace& ns, + const jsg::CompilationObserver& observer) { + const capnp::SchemaLoader& schemaLoader = + js.getCapnpSchemaLoader(); + KJ_IF_SOME(schema, schemaLoader.tryGet(typeId)) { + return js.tryCatch([&] { + auto& typeWrapper = TypeWrapper::from(js.v8Isolate); + ns.setDefault(js, + jsg::JsValue(typeWrapper.wrap(js, js.v8Context(), kj::none, schema) + .template As())); + for (auto nested: schema.getProto().getNestedNodes()) { + KJ_IF_SOME(child, schemaLoader.tryGet(nested.getId())) { + switch (child.getProto().which()) { + case capnp::schema::Node::FILE: + case capnp::schema::Node::STRUCT: + case capnp::schema::Node::INTERFACE: { + ns.set(js, nested.getName(), + jsg::JsValue(typeWrapper.wrap(js, js.v8Context(), kj::none, child) + .template As())); + break; + } + case capnp::schema::Node::ENUM: + case capnp::schema::Node::CONST: + case capnp::schema::Node::ANNOTATION: + // These kinds are not implemented and cannot contain further nested scopes, so + // don't generate anything at all for now. + break; + } + } else { + js.v8Isolate->ThrowException( + js.typeError("Invalid or unknown capnp module type identifier")); + return false; + } + } + return true; + }, [&](jsg::Value exception) { + js.v8Isolate->ThrowException(exception.getHandle(js)); + return false; + }); + } else { + // The schema should have been loaded when the Worker::Script was created. + // This likely indicates an internal error of some kind. + js.v8Isolate->ThrowException( + js.typeError("Invalid or unknown capnp module type identifier")); + return false; + } + }, + exports.releaseAsArray()); } } } diff --git a/src/workerd/server/workerd-api.c++ b/src/workerd/server/workerd-api.c++ index cab0d4c26fc..0602483b2f8 100644 --- a/src/workerd/server/workerd-api.c++ +++ b/src/workerd/server/workerd-api.c++ @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -97,6 +98,7 @@ JSG_DECLARE_ISOLATE_TYPE(JsgWorkerdIsolate, EW_BASICS_ISOLATE_TYPES, EW_BLOB_ISOLATE_TYPES, EW_CACHE_ISOLATE_TYPES, + EW_CAPNP_TYPES, EW_CONTAINER_ISOLATE_TYPES, EW_CJS_ISOLATE_TYPES, EW_CRYPTO_ISOLATE_TYPES, @@ -648,7 +650,37 @@ kj::Maybe WorkerdApi::tryCompileModule(jsg::Loc return kj::none; } KJ_CASE_ONEOF(content, Worker::Script::CapnpModule) { - KJ_FAIL_REQUIRE("capnp modules are not yet supported in workerd"); + const capnp::SchemaLoader& schemaLoader = + lock.getCapnpSchemaLoader(); + auto schema = schemaLoader.get(content.typeId); + + auto fileScope = lock.v8Ref(lock.wrap(lock.v8Context(), schema).As()); + kj::Vector exports; + kj::HashMap topLevelDecls; + + for (auto nested: schema.getProto().getNestedNodes()) { + auto child = schemaLoader.get(nested.getId()); + + switch (child.getProto().which()) { + case capnp::schema::Node::FILE: + case capnp::schema::Node::STRUCT: + case capnp::schema::Node::INTERFACE: { + exports.add(nested.getName()); + topLevelDecls.insert( + nested.getName(), lock.v8Ref(lock.wrap(lock.v8Context(), child).As())); + break; + } + case capnp::schema::Node::ENUM: + case capnp::schema::Node::CONST: + case capnp::schema::Node::ANNOTATION: + // These kinds are not implemented and cannot contain further nested scopes, so + // don't generate anything at all for now. + break; + } + } + + return jsg::ModuleRegistry::ModuleInfo(lock, module.name, exports.asPtr().asConst(), + jsg::ModuleRegistry::CapnpModuleInfo(kj::mv(fileScope), kj::mv(topLevelDecls))); } } KJ_UNREACHABLE; From f558392d5c62613821ff4c1e02f42075a24d5088 Mon Sep 17 00:00:00 2001 From: James M Snell Date: Mon, 15 Sep 2025 15:43:13 -0700 Subject: [PATCH 2/2] Consolidate/deduplicate capnp module handling code --- src/workerd/io/worker-modules.h | 128 ++++++++++++++++++----------- src/workerd/server/workerd-api.c++ | 33 +------- 2 files changed, 84 insertions(+), 77 deletions(-) diff --git a/src/workerd/io/worker-modules.h b/src/workerd/io/worker-modules.h index dbc191b7d82..38b88a2af9c 100644 --- a/src/workerd/io/worker-modules.h +++ b/src/workerd/io/worker-modules.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include #include @@ -9,13 +8,78 @@ #include +#include +#include + +// This header provides utilities for setting up the ModuleRegistry for a worker. +// It is meant to be included in only two places; workerd-api.c++ and the equivalent +// file in the internal repo. It is templated on the TypeWrapper and JsgIsolate types. namespace workerd { +namespace api { +class ServiceWorkerGlobalScope; +class CommonJsModuleContext; +} // namespace api WD_STRONG_BOOL(IsPythonWorker); +namespace modules::capnp { +// Helper to iterate over the nested nodes of a schema for capnp modules, filtering +// out the kinds we don't care about. +void filterNestedNodes(const auto& schemaLoader, const auto& schema, auto fn) { + for (auto nested: schema.getProto().getNestedNodes()) { + auto child = schemaLoader.get(nested.getId()); + switch (child.getProto().which()) { + case ::capnp::schema::Node::FILE: + case ::capnp::schema::Node::STRUCT: + case ::capnp::schema::Node::INTERFACE: { + fn(nested.getName(), child); + break; + } + case ::capnp::schema::Node::ENUM: + case ::capnp::schema::Node::CONST: + case ::capnp::schema::Node::ANNOTATION: + // These kinds are not implemented and cannot contain further nested scopes, so + // don't generate anything at all for now. + break; + } + } +} + +// This is used only by the original module registry implementation in both workerd +// and the internal project. It collects the exports and instantiates the exports of +// a capnp module at the same time and returns a ModuleInfo for the original registry. +// The new module registry variation uses a different approach where the exports are +// collected up front by the exports are instantiated lazily when the module is actually +// resolved. +template +jsg::ModuleRegistry::ModuleInfo addCapnpModule( + typename JsgIsolate::Lock& lock, uint64_t typeId, kj::StringPtr name) { + const auto& schemaLoader = lock.template getCapnpSchemaLoader(); + auto schema = schemaLoader.get(typeId); + auto fileScope = lock.v8Ref(lock.wrap(lock.v8Context(), schema).template As()); + kj::Vector exports; + kj::HashMap topLevelDecls; + + filterNestedNodes(schemaLoader, schema, [&](auto name, const auto& child) { + // topLevelDecls are the actual exported values... + topLevelDecls.insert( + name, lock.v8Ref(lock.wrap(lock.v8Context(), child).template As())); + // ... while exports is just the list of names + exports.add(name); + }); + + return jsg::ModuleRegistry::ModuleInfo(lock, name, exports.asPtr().asConst(), + jsg::ModuleRegistry::CapnpModuleInfo(kj::mv(fileScope), kj::mv(topLevelDecls))); +} +} // namespace modules::capnp + // Creates an instance of the (new) ModuleRegistry. This method provides the // initialization logic that is agnostic to the Worker::Api implementation, // but accepts a callback parameter to handle the Worker::Api-specific details. +// +// Note: this is a big template but it will only be called from two places in +// the codebase, one for workerd and one for the internal project. It depends +// on the TypeWrapper specific to each project. template static kj::Arc newWorkerModuleRegistry( const jsg::ResolveObserver& resolveObserver, @@ -139,63 +203,35 @@ static kj::Arc newWorkerModuleRegistry( break; } KJ_CASE_ONEOF(content, Worker::Script::CapnpModule) { + // For the new module registry, the implementation is a bit different than + // the original. Up front we collect only the names of the exports since we + // need to know those when we create the synthetic module. The actual exports + // themselves, however, are instantiated lazily when the module is actually + // resolved and evaluated. auto& schemaLoader = builder.getSchemaLoader(); auto schema = schemaLoader.get(content.typeId); kj::Vector exports; - for (auto nested: schema.getProto().getNestedNodes()) { - auto child = schemaLoader.get(nested.getId()); - switch (child.getProto().which()) { - case capnp::schema::Node::FILE: - case capnp::schema::Node::STRUCT: - case capnp::schema::Node::INTERFACE: { - exports.add(kj::str(nested.getName())); - break; - } - case capnp::schema::Node::ENUM: - case capnp::schema::Node::CONST: - case capnp::schema::Node::ANNOTATION: - // These kinds are not implemented and cannot contain further nested scopes, so - // don't generate anything at all for now. - break; - } - } + modules::capnp::filterNestedNodes(schemaLoader, schema, + [&](auto name, const capnp::Schema& child) { exports.add(kj::str(name)); }); bundleBuilder.addSyntheticModule(def.name, - [typeId = content.typeId](jsg::Lock& js, const jsg::Url&, + [typeId = content.typeId, &schemaLoader](jsg::Lock& js, const jsg::Url&, const jsg::modules::Module::ModuleNamespace& ns, const jsg::CompilationObserver& observer) { - const capnp::SchemaLoader& schemaLoader = - js.getCapnpSchemaLoader(); + auto& typeWrapper = TypeWrapper::from(js.v8Isolate); KJ_IF_SOME(schema, schemaLoader.tryGet(typeId)) { return js.tryCatch([&] { - auto& typeWrapper = TypeWrapper::from(js.v8Isolate); + // Set the default export... ns.setDefault(js, jsg::JsValue(typeWrapper.wrap(js, js.v8Context(), kj::none, schema) .template As())); - for (auto nested: schema.getProto().getNestedNodes()) { - KJ_IF_SOME(child, schemaLoader.tryGet(nested.getId())) { - switch (child.getProto().which()) { - case capnp::schema::Node::FILE: - case capnp::schema::Node::STRUCT: - case capnp::schema::Node::INTERFACE: { - ns.set(js, nested.getName(), - jsg::JsValue(typeWrapper.wrap(js, js.v8Context(), kj::none, child) - .template As())); - break; - } - case capnp::schema::Node::ENUM: - case capnp::schema::Node::CONST: - case capnp::schema::Node::ANNOTATION: - // These kinds are not implemented and cannot contain further nested scopes, so - // don't generate anything at all for now. - break; - } - } else { - js.v8Isolate->ThrowException( - js.typeError("Invalid or unknown capnp module type identifier")); - return false; - } - } + // Set each of the named exports... + // The names must match what we collected when the bundle was built. + modules::capnp::filterNestedNodes( + schemaLoader, schema, [&](auto name, const auto& child) { + ns.set(js, name, + jsg::JsValue(typeWrapper.wrap(js, js.v8Context(), kj::none, child))); + }); return true; }, [&](jsg::Value exception) { js.v8Isolate->ThrowException(exception.getHandle(js)); diff --git a/src/workerd/server/workerd-api.c++ b/src/workerd/server/workerd-api.c++ index 0602483b2f8..ff86a3d4777 100644 --- a/src/workerd/server/workerd-api.c++ +++ b/src/workerd/server/workerd-api.c++ @@ -650,37 +650,8 @@ kj::Maybe WorkerdApi::tryCompileModule(jsg::Loc return kj::none; } KJ_CASE_ONEOF(content, Worker::Script::CapnpModule) { - const capnp::SchemaLoader& schemaLoader = - lock.getCapnpSchemaLoader(); - auto schema = schemaLoader.get(content.typeId); - - auto fileScope = lock.v8Ref(lock.wrap(lock.v8Context(), schema).As()); - kj::Vector exports; - kj::HashMap topLevelDecls; - - for (auto nested: schema.getProto().getNestedNodes()) { - auto child = schemaLoader.get(nested.getId()); - - switch (child.getProto().which()) { - case capnp::schema::Node::FILE: - case capnp::schema::Node::STRUCT: - case capnp::schema::Node::INTERFACE: { - exports.add(nested.getName()); - topLevelDecls.insert( - nested.getName(), lock.v8Ref(lock.wrap(lock.v8Context(), child).As())); - break; - } - case capnp::schema::Node::ENUM: - case capnp::schema::Node::CONST: - case capnp::schema::Node::ANNOTATION: - // These kinds are not implemented and cannot contain further nested scopes, so - // don't generate anything at all for now. - break; - } - } - - return jsg::ModuleRegistry::ModuleInfo(lock, module.name, exports.asPtr().asConst(), - jsg::ModuleRegistry::CapnpModuleInfo(kj::mv(fileScope), kj::mv(topLevelDecls))); + return workerd::modules::capnp::addCapnpModule( + lock, content.typeId, module.name); } } KJ_UNREACHABLE;