Skip to content

Commit

Permalink
feat: Extract comments from Java code (#2788)
Browse files Browse the repository at this point in the history
Extract comments from Java code for Verbs, Data, Enum, Config, and
Secrets

Closes #2417 

Schema for the test data module is:
```
module javacomments {
  // Config comment
  config config String
  // Secret comment
  secret secretString String
  
  // Comment on a data class.
  export data DataClass {
    field String
  }
  export data EnumType {
    name String
    ordinal Int
  }
  
  // Comment on a verb
  export verb MultilineCommentVerb(javacomments.DataClass) javacomments.EnumType
}
```
  • Loading branch information
tomdaffurn authored Sep 24, 2024
1 parent c910edd commit 093902a
Show file tree
Hide file tree
Showing 9 changed files with 218 additions and 34 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package xyz.block.ftl.deployment;

public class CommentKey {
public static String ofVerb(String verb) {
return "verb." + verb;
}

public static String ofData(String data) {
return "data." + data;
}

public static String ofEnum(String enumName) {
return "enum." + enumName;
}

public static String ofConfig(String config) {
return "config." + config;
}

public static String ofSecret(String secret) {
return "secret." + secret;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.BiFunction;
import java.util.function.Consumer;
Expand Down Expand Up @@ -55,7 +56,6 @@
import xyz.block.ftl.v1.schema.MetadataCalls;
import xyz.block.ftl.v1.schema.MetadataTypeMap;
import xyz.block.ftl.v1.schema.Module;
import xyz.block.ftl.v1.schema.Optional;
import xyz.block.ftl.v1.schema.Ref;
import xyz.block.ftl.v1.schema.Time;
import xyz.block.ftl.v1.schema.Type;
Expand Down Expand Up @@ -84,11 +84,11 @@ public class ModuleBuilder {
private final Map<DotName, TopicsBuildItem.DiscoveredTopic> knownTopics;
private final Map<DotName, VerbClientBuildItem.DiscoveredClients> verbClients;
private final FTLRecorder recorder;
private final Map<String, String> verbDocs;
private final Map<String, Iterable<String>> comments;

public ModuleBuilder(IndexView index, String moduleName, Map<DotName, TopicsBuildItem.DiscoveredTopic> knownTopics,
Map<DotName, VerbClientBuildItem.DiscoveredClients> verbClients, FTLRecorder recorder,
Map<String, String> verbDocs, Map<TypeKey, ExistingRef> typeAliases) {
Map<String, Iterable<String>> comments, Map<TypeKey, ExistingRef> typeAliases) {
this.index = index;
this.moduleName = moduleName;
this.moduleBuilder = Module.newBuilder()
Expand All @@ -97,7 +97,7 @@ public ModuleBuilder(IndexView index, String moduleName, Map<DotName, TopicsBuil
this.knownTopics = knownTopics;
this.verbClients = verbClients;
this.recorder = recorder;
this.verbDocs = verbDocs;
this.comments = comments;
this.dataElements = new HashMap<>(typeAliases);
}

Expand Down Expand Up @@ -184,8 +184,12 @@ public void registerVerbMethod(MethodInfo method, String className,
String name = param.annotation(Secret.class).value().asString();
paramMappers.add(new VerbRegistry.SecretSupplier(name, paramType));
if (!knownSecrets.contains(name)) {
addDecls(Decl.newBuilder().setSecret(xyz.block.ftl.v1.schema.Secret.newBuilder()
.setType(buildType(param.type(), false)).setName(name)).build());
xyz.block.ftl.v1.schema.Secret.Builder secretBuilder = xyz.block.ftl.v1.schema.Secret.newBuilder()
.setType(buildType(param.type(), false))
.setName(name);
Optional.ofNullable(comments.get(CommentKey.ofSecret(name)))
.ifPresent(secretBuilder::addAllComments);
addDecls(Decl.newBuilder().setSecret(secretBuilder).build());
knownSecrets.add(name);
}
} else if (param.hasAnnotation(Config.class)) {
Expand All @@ -194,8 +198,12 @@ public void registerVerbMethod(MethodInfo method, String className,
String name = param.annotation(Config.class).value().asString();
paramMappers.add(new VerbRegistry.ConfigSupplier(name, paramType));
if (!knownConfig.contains(name)) {
addDecls(Decl.newBuilder().setConfig(xyz.block.ftl.v1.schema.Config.newBuilder()
.setType(buildType(param.type(), false)).setName(name)).build());
xyz.block.ftl.v1.schema.Config.Builder configBuilder = xyz.block.ftl.v1.schema.Config.newBuilder()
.setType(buildType(param.type(), false))
.setName(name);
Optional.ofNullable(comments.get(CommentKey.ofConfig(name)))
.ifPresent(configBuilder::addAllComments);
addDecls(Decl.newBuilder().setConfig(configBuilder).build());
knownConfig.add(name);
}
} else if (knownTopics.containsKey(param.type().name())) {
Expand Down Expand Up @@ -242,9 +250,8 @@ public void registerVerbMethod(MethodInfo method, String className,
.setExport(exported)
.setRequest(buildType(bodyParamType, exported))
.setResponse(buildType(method.returnType(), exported));
if (verbDocs.containsKey(verbName)) {
verbBuilder.addComments(verbDocs.get(verbName));
}
Optional.ofNullable(comments.get(CommentKey.ofVerb(verbName)))
.ifPresent(verbBuilder::addAllComments);

if (metadataCallback != null) {
metadataCallback.accept(verbBuilder);
Expand Down Expand Up @@ -301,7 +308,9 @@ public Type buildType(org.jboss.jandex.Type type, boolean export) {
if (type.hasAnnotation(NOT_NULL)) {
return primitive;
}
return Type.newBuilder().setOptional(Optional.newBuilder().setType(primitive)).build();
return Type.newBuilder().setOptional(xyz.block.ftl.v1.schema.Optional.newBuilder()
.setType(primitive))
.build();
}
if (info != null && info.hasDeclaredAnnotation(GENERATED_REF)) {
var ref = info.declaredAnnotation(GENERATED_REF);
Expand Down Expand Up @@ -347,6 +356,8 @@ public Type buildType(org.jboss.jandex.Type type, boolean export) {
Data.Builder data = Data.newBuilder();
data.setName(clazz.name().local());
data.setExport(type.hasAnnotation(EXPORT) || export);
Optional.ofNullable(comments.get(CommentKey.ofData(clazz.name().local())))
.ifPresent(data::addAllComments);
buildDataElement(data, clazz.name());
moduleBuilder.addDecls(Decl.newBuilder().setData(data).build());
Ref ref = Ref.newBuilder().setName(data.getName()).setModule(moduleName).build();
Expand All @@ -367,9 +378,8 @@ public Type buildType(org.jboss.jandex.Type type, boolean export) {
.build();
} else if (paramType.name().equals(DotNames.OPTIONAL)) {
//TODO: optional kinda sucks
return Type.newBuilder()
.setOptional(
Optional.newBuilder().setType(buildType(paramType.arguments().get(0), export)))
return Type.newBuilder().setOptional(xyz.block.ftl.v1.schema.Optional.newBuilder()
.setType(buildType(paramType.arguments().get(0), export)))
.build();
} else if (paramType.name().equals(DotName.createSimple(HttpRequest.class))) {
return Type.newBuilder()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.PosixFilePermission;
import java.util.Arrays;
import java.util.Base64;
import java.util.EnumSet;
import java.util.HashMap;
Expand Down Expand Up @@ -113,21 +114,7 @@ public void generateSchema(CombinedIndexBuildItem index,
List<TypeAliasBuildItem> typeAliasBuildItems,
List<SchemaContributorBuildItem> schemaContributorBuildItems) throws Exception {
String moduleName = moduleNameBuildItem.getModuleName();
Map<String, String> verbDocs = new HashMap<>();
try (var input = Thread.currentThread().getContextClassLoader().getResourceAsStream("META-INF/ftl-verbs.txt")) {
if (input != null) {
var contents = new String(input.readAllBytes(), StandardCharsets.UTF_8).split("\n");
for (var content : contents) {
var eq = content.indexOf('=');
if (eq == -1) {
continue;
}
String key = content.substring(0, eq);
String value = new String(Base64.getDecoder().decode(content.substring(eq + 1)), StandardCharsets.UTF_8);
verbDocs.put(key, value);
}
}
}
Map<String, Iterable<String>> comments = readComments();
Map<TypeKey, ModuleBuilder.ExistingRef> existingRefs = new HashMap<>();
for (var i : typeAliasBuildItems) {
String mn;
Expand All @@ -146,7 +133,7 @@ public void generateSchema(CombinedIndexBuildItem index,
}

ModuleBuilder moduleBuilder = new ModuleBuilder(index.getComputingIndex(), moduleName, topicsBuildItem.getTopics(),
verbClientBuildItem.getVerbClients(), recorder, verbDocs, existingRefs);
verbClientBuildItem.getVerbClients(), recorder, comments, existingRefs);

for (var i : schemaContributorBuildItems) {
i.getSchemaContributor().accept(moduleBuilder);
Expand Down Expand Up @@ -201,4 +188,27 @@ void openSocket(BuildProducer<RequireVirtualHttpBuildItem> virtual,
socket.produce(RequireSocketHttpBuildItem.MARKER);
virtual.produce(RequireVirtualHttpBuildItem.MARKER);
}

/**
* Bytecode doesn't retain comments, so they are stored in a separate file
* Each line is a key value pair separated by an '='. The key is the DeclRef and the value is the comment
*/
private Map<String, Iterable<String>> readComments() throws IOException {
Map<String, Iterable<String>> comments = new HashMap<>();
try (var input = Thread.currentThread().getContextClassLoader().getResourceAsStream("META-INF/ftl-verbs.txt")) {
if (input != null) {
var contents = new String(input.readAllBytes(), StandardCharsets.UTF_8).split("\n");
for (var content : contents) {
var eq = content.indexOf('=');
if (eq == -1) {
continue;
}
String key = content.substring(0, eq);
String value = new String(Base64.getDecoder().decode(content.substring(eq + 1)), StandardCharsets.UTF_8);
comments.put(key, Arrays.asList(value.split("\n")));
}
}
}
return comments;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,19 +19,28 @@
import javax.lang.model.SourceVersion;
import javax.lang.model.element.AnnotationMirror;
import javax.lang.model.element.Element;
import javax.lang.model.element.ElementKind;
import javax.lang.model.element.ExecutableElement;
import javax.lang.model.element.TypeElement;
import javax.lang.model.element.VariableElement;
import javax.tools.Diagnostic;
import javax.tools.FileObject;
import javax.tools.StandardLocation;

import xyz.block.ftl.Config;
import xyz.block.ftl.Export;
import xyz.block.ftl.Secret;
import xyz.block.ftl.Verb;

/**
* POC annotation processor for capturing JavaDoc, this needs a lot more work.
*/
public class AnnotationProcessor implements Processor {
private static final Pattern REMOVE_LEADING_SPACE = Pattern.compile("^ ", Pattern.MULTILINE);
private static final Pattern REMOVE_JAVADOC_TAGS = Pattern.compile(
"^\\s*@(param|return|throws|exception|see|author)\\b[^\\n]*$\\n*",
Pattern.MULTILINE);

private ProcessingEnvironment processingEnv;

final Map<String, String> saved = new HashMap<>();
Expand All @@ -43,7 +52,7 @@ public Set<String> getSupportedOptions() {

@Override
public Set<String> getSupportedAnnotationTypes() {
return Set.of(Verb.class.getName());
return Set.of(Verb.class.getName(), Export.class.getName());
}

@Override
Expand All @@ -59,10 +68,36 @@ public void init(ProcessingEnvironment processingEnv) {
@Override
public boolean process(Set<? extends TypeElement> annotations, RoundEnvironment roundEnv) {
//TODO: @VerbName, HTTP, CRON etc
roundEnv.getElementsAnnotatedWith(Verb.class)
roundEnv.getElementsAnnotatedWithAny(Set.of(Verb.class, Export.class))
.forEach(element -> {
Optional<String> javadoc = getJavadoc(element);
javadoc.ifPresent(doc -> saved.put(element.getSimpleName().toString(), doc));

javadoc.ifPresent(doc -> {
String strippedDownDoc = stripJavadocTags(doc);
String key = element.getSimpleName().toString();

if (element.getKind() == ElementKind.METHOD) {
saved.put("verb." + key, strippedDownDoc);
} else if (element.getKind() == ElementKind.CLASS) {
saved.put("data." + key, strippedDownDoc);
} else if (element.getKind() == ElementKind.ENUM) {
saved.put("enum." + key, strippedDownDoc);
}

if (element.getKind() == ElementKind.METHOD) {
var executableElement = (ExecutableElement) element;
executableElement.getParameters().forEach(param -> {
Config config = param.getAnnotation(Config.class);
if (config != null) {
saved.put("config." + config.value(), extractCommentForParam(doc, param));
}
Secret secret = param.getAnnotation(Secret.class);
if (secret != null) {
saved.put("secret." + secret.value(), extractCommentForParam(doc, param));
}
});
}
});
});

if (roundEnv.processingOver()) {
Expand Down Expand Up @@ -117,4 +152,25 @@ public Optional<String> getJavadoc(Element e) {
.trim());
}

public String stripJavadocTags(String doc) {
// TODO extract JavaDoc tags to a rich markdown model supported by schema
return REMOVE_JAVADOC_TAGS.matcher(doc).replaceAll("");
}

/**
* Read the @param tag in a JavaDoc comment to extract Config and Secret comments
*/
private String extractCommentForParam(String doc, VariableElement param) {
String variableName = param.getSimpleName().toString();
int startIdx = doc.indexOf("@param " + variableName + " ");
if (startIdx != -1) {
int endIndex = doc.indexOf("\n", startIdx);
if (endIndex == -1) {
endIndex = doc.length();
}
return doc.substring(startIdx + variableName.length() + 8, endIndex);
}
return null;
}

}
2 changes: 2 additions & 0 deletions jvm-runtime/testdata/java/javacomments/ftl.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
module = "javacomments"
language = "java"
22 changes: 22 additions & 0 deletions jvm-runtime/testdata/java/javacomments/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>xyz.block.ftl.examples</groupId>
<artifactId>javacomments</artifactId>
<version>1.0-SNAPSHOT</version>

<parent>
<groupId>xyz.block.ftl</groupId>
<artifactId>ftl-build-parent-java</artifactId>
<version>1.0-SNAPSHOT</version>
</parent>

<dependencies>
<dependency>
<groupId>xyz.block</groupId>
<artifactId>web5-dids</artifactId>
<version>2.0.1-debug1</version>
</dependency>
</dependencies>

</project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package xyz.block.ftl.javacomments;

import org.jetbrains.annotations.NotNull;

import xyz.block.ftl.Config;
import xyz.block.ftl.Export;
import xyz.block.ftl.Secret;
import xyz.block.ftl.Verb;

/**
* Comment on a module class
*/
public class CommentedModule {

/**
* Comment on a verb
*
* @param val Parameter comment
* @param configString Config comment
* @param secretString Secret comment
* @return Great success
*/
@Export
@Verb
public @NotNull EnumType MultilineCommentVerb(
@NotNull DataClass val,
@Config("config") String configString,
@Secret("secretString") String secretString) {
return EnumType.PORTENTOUS;
}

//TODO TypeAlias, Database, Topic, Subscription, Lease, Cron
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package xyz.block.ftl.javacomments;

import xyz.block.ftl.Export;

/**
* Comment on a data class.
*/
@Export
public class DataClass {
/**
* Comment on a data field.
*/
private String field;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package xyz.block.ftl.javacomments;

import xyz.block.ftl.Export;

/**
* Comment on an enum type
*/
@Export
public enum EnumType {
/**
* Comment on an enum value
*/
PORTENTOUS
}

0 comments on commit 093902a

Please sign in to comment.