Implement 'extern library' support for functions. (#4220)

Support for types (particularly classes) is left as a TODO.

There's also an issue I'm observing with a "define in impl" test, but
this is probably an issue with resolving the prior declaration which is
imported indirectly. The PR was already feeling big, so I'm choosing to
cut here.

Note, this does not implement the rule "The owning library's API file
must import the `extern` declaration, and must also contain a
declaration."

toolchain/check/decl_introducer_state.h

@@ -35,6 +35,9 @@ struct DeclIntroducerState {
 
   // Invariant: contains just the modifiers represented by `saw_*_modifier`.
   KeywordModifierSet modifier_set = KeywordModifierSet();
+
+  // If there's an `extern library` in use, the library name.
+  SemIR::LibraryNameId extern_library = SemIR::LibraryNameId::Invalid;
 };
 
 // Stack of `DeclIntroducerState` values, representing all the declaration

toolchain/check/decl_name_stack.h

@@ -91,7 +91,8 @@ class DeclNameStack {
     // Combines name information to produce a base struct for entity
     // construction.
     auto MakeEntityWithParamsBase(const NameComponent& name,
-                                  SemIR::InstId decl_id, bool is_extern)
+                                  SemIR::InstId decl_id, bool is_extern,
+                                  SemIR::LibraryNameId extern_library)
         -> SemIR::EntityWithParamsBase {
       return {
           .name_id = name_id_for_new_inst(),
@@ -102,9 +103,11 @@ class DeclNameStack {
           .implicit_param_refs_id = name.implicit_params_id,
           .param_refs_id = name.params_id,
           .is_extern = is_extern,
-          .extern_library_id = StringLiteralValueId::Invalid,
-          .non_owning_decl_id = SemIR::InstId::Invalid,
-          .first_owning_decl_id = decl_id,
+          .extern_library_id = extern_library,
+          .non_owning_decl_id =
+              extern_library.is_valid() ? decl_id : SemIR::InstId::Invalid,
+          .first_owning_decl_id =
+              extern_library.is_valid() ? SemIR::InstId::Invalid : decl_id,
       };
     }
 

toolchain/check/global_init.cpp

@@ -43,7 +43,7 @@ auto GlobalInit::Finalize() -> void {
         .implicit_param_refs_id = SemIR::InstBlockId::Invalid,
         .param_refs_id = SemIR::InstBlockId::Empty,
         .is_extern = false,
-        .extern_library_id = StringLiteralValueId::Invalid,
+        .extern_library_id = SemIR::LibraryNameId::Invalid,
         .non_owning_decl_id = SemIR::InstId::Invalid,
         .first_owning_decl_id = SemIR::InstId::Invalid},
        {.return_storage_id = SemIR::InstId::Invalid,

toolchain/check/handle_class.cpp

@@ -52,8 +52,8 @@ auto HandleParseNode(Context& context, Parse::ClassIntroducerId node_id)
 // Otherwise, returns false. Prints a diagnostic when appropriate.
 static auto MergeClassRedecl(Context& context, SemIRLoc new_loc,
                              SemIR::Class& new_class, bool new_is_import,
-                             bool new_is_definition, bool new_is_extern,
-                             SemIR::ClassId prev_class_id, bool prev_is_extern,
+                             bool new_is_definition,
+                             SemIR::ClassId prev_class_id,
                              SemIR::ImportIRId prev_import_ir_id) -> bool {
   auto& prev_class = context.classes().Get(prev_class_id);
   SemIRLoc prev_loc = prev_class.latest_decl_id();
@@ -64,14 +64,11 @@ static auto MergeClassRedecl(Context& context, SemIRLoc new_loc,
     return false;
   }
 
-  CheckIsAllowedRedecl(context, Lex::TokenKind::Class, prev_class.name_id,
-                       {.loc = new_loc,
-                        .is_definition = new_is_definition,
-                        .is_extern = new_is_extern},
-                       {.loc = prev_loc,
-                        .is_definition = prev_class.is_defined(),
-                        .is_extern = prev_is_extern},
-                       prev_import_ir_id);
+  CheckIsAllowedRedecl(
+      context, Lex::TokenKind::Class, prev_class.name_id,
+      RedeclInfo(new_class, new_loc, new_is_definition),
+      RedeclInfo(prev_class, prev_loc, prev_class.is_defined()),
+      prev_import_ir_id);
 
   if (new_is_definition && prev_class.is_defined()) {
     // Don't attempt to merge multiple definitions.
@@ -101,7 +98,7 @@ static auto MergeClassRedecl(Context& context, SemIRLoc new_loc,
   }
 
   if ((prev_import_ir_id.is_valid() && !new_is_import) ||
-      (prev_is_extern && !new_is_extern)) {
+      (prev_class.is_extern && !new_class.is_extern)) {
     prev_class.first_owning_decl_id = new_class.first_owning_decl_id;
     ReplacePrevInstForMerge(
         context, prev_class.parent_scope_id, prev_class.name_id,
@@ -117,8 +114,7 @@ static auto MergeOrAddName(Context& context, Parse::AnyClassDeclId node_id,
                            SemIR::InstId class_decl_id,
                            SemIR::ClassDecl& class_decl,
                            SemIR::Class& class_info, bool is_definition,
-                           bool is_extern, SemIR::AccessKind access_kind)
-    -> void {
+                           SemIR::AccessKind access_kind) -> void {
   auto prev_id = context.decl_name_stack().LookupOrAddName(
       name_context, class_decl_id, access_kind);
   if (!prev_id.is_valid()) {
@@ -168,10 +164,10 @@ static auto MergeOrAddName(Context& context, Parse::AnyClassDeclId node_id,
     return;
   }
 
-  // TODO: Fix prev_is_extern logic.
+  // TODO: Fix `extern` logic. It doesn't work correctly, but doesn't seem worth
+  // ripping out because existing code may incrementally help.
   if (MergeClassRedecl(context, node_id, class_info,
-                       /*new_is_import=*/false, is_definition, is_extern,
-                       prev_class_id, /*prev_is_extern=*/false,
+                       /*new_is_import=*/false, is_definition, prev_class_id,
                        prev_import_ir_id)) {
     // When merging, use the existing entity rather than adding a new one.
     class_decl.class_id = prev_class_id;
@@ -201,6 +197,9 @@ static auto BuildClassDecl(Context& context, Parse::AnyClassDeclId node_id,
                                is_definition);
 
   bool is_extern = introducer.modifier_set.HasAnyOf(KeywordModifierSet::Extern);
+  if (introducer.extern_library.is_valid()) {
+    context.TODO(node_id, "extern library");
+  }
   auto inheritance_kind =
       introducer.modifier_set.HasAnyOf(KeywordModifierSet::Abstract)
           ? SemIR::Class::Abstract
@@ -219,7 +218,8 @@ static auto BuildClassDecl(Context& context, Parse::AnyClassDeclId node_id,
 
   // TODO: Store state regarding is_extern.
   SemIR::Class class_info = {
-      name_context.MakeEntityWithParamsBase(name, class_decl_id, is_extern),
+      name_context.MakeEntityWithParamsBase(name, class_decl_id, is_extern,
+                                            SemIR::LibraryNameId::Invalid),
       {// `.self_type_id` depends on the ClassType, so is set below.
        .self_type_id = SemIR::TypeId::Invalid,
        .inheritance_kind = inheritance_kind}};
@@ -228,7 +228,7 @@ static auto BuildClassDecl(Context& context, Parse::AnyClassDeclId node_id,
   RequireGenericParams(context, class_info.param_refs_id);
 
   MergeOrAddName(context, node_id, name_context, class_decl_id, class_decl,
-                 class_info, is_definition, is_extern,
+                 class_info, is_definition,
                  introducer.modifier_set.GetAccessKind());
 
   // Create a new class if this isn't a valid redeclaration.

toolchain/check/handle_function.cpp

@@ -82,12 +82,9 @@ static auto MergeFunctionRedecl(Context& context, SemIRLoc new_loc,
   }
 
   CheckIsAllowedRedecl(context, Lex::TokenKind::Fn, prev_function.name_id,
-                       {.loc = new_loc,
-                        .is_definition = new_is_definition,
-                        .is_extern = new_function.is_extern},
-                       {.loc = prev_function.latest_decl_id(),
-                        .is_definition = prev_function.definition_id.is_valid(),
-                        .is_extern = prev_function.is_extern},
+                       RedeclInfo(new_function, new_loc, new_is_definition),
+                       RedeclInfo(prev_function, prev_function.latest_decl_id(),
+                                  prev_function.definition_id.is_valid()),
                        prev_import_ir_id);
 
   if (new_is_definition) {
@@ -216,9 +213,10 @@ static auto BuildFunctionDecl(Context& context,
 
   // Build the function entity. This will be merged into an existing function if
   // there is one, or otherwise added to the function store.
-  auto function_info = SemIR::Function{
-      {name_context.MakeEntityWithParamsBase(name, decl_id, is_extern)},
-      {.return_storage_id = return_storage_id}};
+  auto function_info =
+      SemIR::Function{{name_context.MakeEntityWithParamsBase(
+                          name, decl_id, is_extern, introducer.extern_library)},
+                      {.return_storage_id = return_storage_id}};
   if (is_definition) {
     function_info.definition_id = decl_id;
   }

toolchain/check/handle_import_and_package.cpp

@@ -20,6 +20,7 @@ auto HandleParseNode(Context& context, Parse::ImportIntroducerId /*node_id*/)
 
 auto HandleParseNode(Context& context, Parse::ImportDeclId /*node_id*/)
     -> bool {
+  context.node_stack().PopIf<SemIR::LibraryNameId>();
   auto introducer =
       context.decl_introducer_state_stack().Pop<Lex::TokenKind::Import>();
   LimitModifiersOnDecl(context, introducer, KeywordModifierSet::Export);
@@ -34,6 +35,7 @@ auto HandleParseNode(Context& context, Parse::LibraryIntroducerId /*node_id*/)
 
 auto HandleParseNode(Context& context, Parse::LibraryDeclId /*node_id*/)
     -> bool {
+  context.node_stack().PopIf<SemIR::LibraryNameId>();
   auto introducer =
       context.decl_introducer_state_stack().Pop<Lex::TokenKind::Library>();
   LimitModifiersOnDecl(context, introducer, KeywordModifierSet::Impl);
@@ -48,14 +50,16 @@ auto HandleParseNode(Context& context, Parse::PackageIntroducerId /*node_id*/)
 
 auto HandleParseNode(Context& context, Parse::PackageDeclId /*node_id*/)
     -> bool {
+  context.node_stack().PopIf<SemIR::LibraryNameId>();
   auto introducer =
       context.decl_introducer_state_stack().Pop<Lex::TokenKind::Package>();
   LimitModifiersOnDecl(context, introducer, KeywordModifierSet::Impl);
   return true;
 }
 
-auto HandleParseNode(Context& /*context*/,
-                     Parse::LibrarySpecifierId /*node_id*/) -> bool {
+auto HandleParseNode(Context& context, Parse::LibrarySpecifierId /*node_id*/)
+    -> bool {
+  CARBON_CHECK(context.node_stack().PeekIs<SemIR::LibraryNameId>());
   return true;
 }
 
@@ -64,13 +68,19 @@ auto HandleParseNode(Context& /*context*/, Parse::PackageNameId /*node_id*/)
   return true;
 }
 
-auto HandleParseNode(Context& /*context*/, Parse::LibraryNameId /*node_id*/)
-    -> bool {
+auto HandleParseNode(Context& context, Parse::LibraryNameId node_id) -> bool {
+  // This is discarded in this file's uses, but is used by modifiers for `extern
+  // library`.
+  auto literal_id = context.tokens().GetStringLiteralValue(
+      context.parse_tree().node_token(node_id));
+  context.node_stack().Push(
+      node_id, SemIR::LibraryNameId::ForStringLiteralValueId(literal_id));
   return true;
 }
 
-auto HandleParseNode(Context& /*context*/, Parse::DefaultLibraryId /*node_id*/)
+auto HandleParseNode(Context& context, Parse::DefaultLibraryId node_id)
     -> bool {
+  context.node_stack().Push(node_id, SemIR::LibraryNameId::Default);
   return true;
 }
 

toolchain/check/handle_interface.cpp

@@ -54,7 +54,8 @@ static auto BuildInterfaceDecl(Context& context,
       context.AddPlaceholderInst(SemIR::LocIdAndInst(node_id, interface_decl));
 
   SemIR::Interface interface_info = {name_context.MakeEntityWithParamsBase(
-      name, interface_decl_id, /*is_extern=*/false)};
+      name, interface_decl_id, /*is_extern=*/false,
+      SemIR::LibraryNameId::Invalid)};
   RequireGenericParams(context, interface_info.implicit_param_refs_id);
   RequireGenericParams(context, interface_info.param_refs_id);
 
@@ -75,15 +76,12 @@ static auto BuildInterfaceDecl(Context& context,
         // TODO: This should be refactored a little, particularly for
         // prev_import_ir_id. See similar logic for classes and functions, which
         // might also be refactored to merge.
-        CheckIsAllowedRedecl(context, Lex::TokenKind::Interface,
-                             existing_interface.name_id,
-                             {.loc = node_id,
-                              .is_definition = is_definition,
-                              .is_extern = false},
-                             {.loc = existing_interface.latest_decl_id(),
-                              .is_definition = existing_interface.is_defined(),
-                              .is_extern = false},
-                             /*prev_import_ir_id=*/SemIR::ImportIRId::Invalid);
+        CheckIsAllowedRedecl(
+            context, Lex::TokenKind::Interface, existing_interface.name_id,
+            RedeclInfo(interface_info, node_id, is_definition),
+            RedeclInfo(existing_interface, existing_interface.latest_decl_id(),
+                       existing_interface.is_defined()),
+            /*prev_import_ir_id=*/SemIR::ImportIRId::Invalid);
 
         // Can't merge interface definitions due to the generic requirements.
         // TODO: Should this also be mirrored to classes/functions for generics?

toolchain/check/handle_modifier.cpp

@@ -34,6 +34,10 @@ static auto DiagnoseNotAllowedWith(Context& context, Parse::NodeId first_node,
       .Emit();
 }
 
+// Handles the keyword that starts a modifier. This may a standalone keyword,
+// such as `private`, or the first in a complex modifier, such as `extern` in
+// `extern library ...`. If valid, adds it to the modifier set and returns true.
+// Otherwise, diagnoses and returns false.
 static auto HandleModifier(Context& context, Parse::NodeId node_id,
                            KeywordModifierSet keyword) -> bool {
   auto& s = context.decl_introducer_state_stack().innermost();
@@ -54,10 +58,14 @@ static auto HandleModifier(Context& context, Parse::NodeId node_id,
   auto current_modifier_node_id = s.modifier_node_id(order);
   if (s.modifier_set.HasAnyOf(keyword)) {
     DiagnoseRepeated(context, current_modifier_node_id, node_id);
-  } else if (current_modifier_node_id.is_valid()) {
+    return false;
+  }
+  if (current_modifier_node_id.is_valid()) {
     DiagnoseNotAllowedWith(context, current_modifier_node_id, node_id);
-  } else if (auto later_modifier_set = s.modifier_set & later_modifiers;
-             !later_modifier_set.empty()) {
+    return false;
+  }
+  if (auto later_modifier_set = s.modifier_set & later_modifiers;
+      !later_modifier_set.empty()) {
     // At least one later modifier is present. Diagnose using the closest.
     Parse::NodeId closest_later_modifier = Parse::NodeId::Invalid;
     for (auto later_order = static_cast<int8_t>(order) + 1;
@@ -79,24 +87,31 @@ static auto HandleModifier(Context& context, Parse::NodeId node_id,
         .Note(closest_later_modifier, ModifierPrevious,
               context.token_kind(closest_later_modifier))
         .Emit();
-  } else {
-    s.modifier_set.Add(keyword);
-    s.set_modifier_node_id(order, node_id);
+    return false;
   }
+
+  s.modifier_set.Add(keyword);
+  s.set_modifier_node_id(order, node_id);
   return true;
 }
 
 #define CARBON_PARSE_NODE_KIND(...)
 #define CARBON_PARSE_NODE_KIND_TOKEN_MODIFIER(Name, ...)                  \
   auto HandleParseNode(Context& context, Parse::Name##ModifierId node_id) \
       -> bool {                                                           \
-    return HandleModifier(context, node_id, KeywordModifierSet::Name);    \
+    HandleModifier(context, node_id, KeywordModifierSet::Name);           \
+    return true;                                                          \
   }
 #include "toolchain/parse/node_kind.def"
 
 auto HandleParseNode(Context& context,
                      Parse::ExternModifierWithLibraryId node_id) -> bool {
-  return context.TODO(node_id, "extern library syntax");
+  auto name_literal_id = context.node_stack().Pop<SemIR::LibraryNameId>();
+  if (HandleModifier(context, node_id, KeywordModifierSet::Extern)) {
+    auto& s = context.decl_introducer_state_stack().innermost();
+    s.extern_library = name_literal_id;
+  }
+  return true;
 }
 
 auto HandleParseNode(Context& context, Parse::ExternModifierId node_id)

toolchain/check/handle_struct.cpp

@@ -29,7 +29,7 @@ auto HandleParseNode(Context& context, Parse::StructLiteralStartId node_id)
 auto HandleParseNode(Context& context,
                      Parse::StructFieldDesignatorId /*node_id*/) -> bool {
   // This leaves the designated name on top because the `.` isn't interesting.
-  CARBON_CHECK(context.node_stack().PeekIsName());
+  CARBON_CHECK(context.node_stack().PeekIs<SemIR::NameId>());
   return true;
 }
 

toolchain/check/import_ref.cpp

@@ -844,6 +844,19 @@ class ImportRefResolver {
   auto GetIncompleteLocalEntityBase(
       SemIR::InstId decl_id, const SemIR::EntityWithParamsBase& import_base)
       -> SemIR::EntityWithParamsBase {
+    // Translate the extern_library_id if present.
+    auto extern_library_id = SemIR::LibraryNameId::Invalid;
+    if (import_base.extern_library_id.is_valid()) {
+      if (import_base.extern_library_id.index >= 0) {
+        auto val = import_ir_.string_literal_values().Get(
+            import_base.extern_library_id.AsStringLiteralValueId());
+        extern_library_id = SemIR::LibraryNameId::ForStringLiteralValueId(
+            context_.string_literal_values().Add(val));
+      } else {
+        extern_library_id = import_base.extern_library_id;
+      }
+    }
+
     return {
         .name_id = GetLocalNameId(import_base.name_id),
         .parent_scope_id = SemIR::NameScopeId::Invalid,
@@ -857,9 +870,13 @@ class ImportRefResolver {
                              ? SemIR::InstBlockId::Empty
                              : SemIR::InstBlockId::Invalid,
         .is_extern = import_base.is_extern,
-        .extern_library_id = StringLiteralValueId::Invalid,
-        .non_owning_decl_id = SemIR::InstId::Invalid,
-        .first_owning_decl_id = decl_id,
+        .extern_library_id = extern_library_id,
+        .non_owning_decl_id = import_base.non_owning_decl_id.is_valid()
+                                  ? decl_id
+                                  : SemIR::InstId::Invalid,
+        .first_owning_decl_id = import_base.first_owning_decl_id.is_valid()
+                                    ? decl_id
+                                    : SemIR::InstId::Invalid,
     };
   }
 
@@ -1373,7 +1390,7 @@ class ImportRefResolver {
         .decl_block_id = SemIR::InstBlockId::Empty};
     auto function_decl_id =
         context_.AddPlaceholderInstInNoBlock(SemIR::LocIdAndInst(
-            AddImportIRInst(import_function.latest_decl_id()), function_decl));
+            AddImportIRInst(import_function.first_decl_id()), function_decl));
 
     // Start with an incomplete function.
     function_decl.function_id = context_.functions().Add(
@@ -1471,7 +1488,7 @@ class ImportRefResolver {
   auto TryResolveTypedInst(SemIR::FunctionType inst) -> ResolveResult {
     CARBON_CHECK(inst.type_id == SemIR::TypeId::TypeType);
     auto fn_val_id = GetLocalConstantInstId(
-        import_ir_.functions().Get(inst.function_id).first_owning_decl_id);
+        import_ir_.functions().Get(inst.function_id).first_decl_id());
     auto specific_data = GetLocalSpecificData(inst.specific_id);
     if (HasNewWork()) {
       return Retry();