Huge speedup by separating "find" from "generate" by nedtwigg · Pull Request #66 · diffplug/atplug · GitHub
Skip to content

Huge speedup by separating "find" from "generate" #66

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nedtwigg merged 6 commits into from
Jan 27, 2025
Merged

Huge speedup by separating "find" from "generate" #66

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
5b04e86
PlugGenerator no longer finds plugs, it only generates what was found…
nedtwigg Jan 27, 2025
8838901
Add a `FindPlugsTask` which does the finding.
nedtwigg Jan 27, 2025
bba5c30
Wire up the new mechanisms.
nedtwigg Jan 27, 2025
8b96ea0
More speedup by reusing the parser buffer.
nedtwigg Jan 27, 2025
53164d0
Better task names to match previous behavior.
nedtwigg Jan 27, 2025
69f9e10
Update changelog.
nedtwigg Jan 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
54 changes: 19 additions & 35 deletions atplug-plugin-gradle/src/main/java/com/diffplug/atplug/tooling/PlugGenerator.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,18 +22,18 @@ import java.lang.UnsatisfiedLinkError
import java.lang.reflect.Constructor
import java.lang.reflect.Modifier
import java.net.URLClassLoader
import java.nio.file.FileVisitResult
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.SimpleFileVisitor
import java.nio.file.attribute.BasicFileAttributes
import java.util.*
import java.util.function.Function
import kotlin.reflect.KFunction
import kotlin.reflect.full.companionObjectInstance
import kotlin.reflect.full.memberFunctions

class PlugGenerator internal constructor(toSearches: List<File>, toLinkAgainst: Set<File>) {
class PlugGenerator
internal constructor(
plugToSocket: Map<String, String>,
toSearches: List<File>,
toLinkAgainst: Set<File>
) {
@JvmField val atplugInf: SortedMap<String, String> = TreeMap()

/** A cache from a plugin interface to a function that converts a class into its metadata. */
Expand All @@ -56,22 +56,7 @@ class PlugGenerator internal constructor(toSearches: List<File>, toLinkAgainst:
}!!
as KFunction<Function<Any, String>>
try {
val parser = PlugParser()
// walk toSearch, passing each classfile to load()
for (toSearch in toSearches) {
if (toSearch.isDirectory) {
Files.walkFileTree(
toSearch.toPath(),
object : SimpleFileVisitor<Path>() {
override fun visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult {
if (file.toString().endsWith(EXT_CLASS)) {
maybeGeneratePlugin(parser, file)
}
return FileVisitResult.CONTINUE
}
})
}
}
plugToSocket.forEach { (plug, socket) -> atplugInf[plug] = generatePlugin(plug, socket) }
} finally {
classLoader.close()
System.setProperty("atplug.generate", "")
Expand All @@ -82,18 +67,13 @@ class PlugGenerator internal constructor(toSearches: List<File>, toLinkAgainst:
* Loads a class by its FQN. If it's concrete and implements a plugin, then we'll call
* generatePlugin.
*/
private fun maybeGeneratePlugin(parser: PlugParser, path: Path) {
parser.parse(path.toFile())
if (!parser.hasPlug()) {
return
}
val plugClass = classLoader.loadClass(parser.plugClassName)
val socketClass = classLoader.loadClass(parser.socketClassName)
private fun generatePlugin(plugClassName: String, socketClassName: String): String {
val plugClass = classLoader.loadClass(plugClassName)
val socketClass = classLoader.loadClass(socketClassName)
require(!Modifier.isAbstract(plugClass.modifiers)) {
"Class $plugClass has @Plug($socketClass) but it is abstract."
}
val atplugInfContent = generatePlugin<Any, Any>(plugClass, socketClass)
atplugInf[plugClass.name] = atplugInfContent
return generatePlugin<Any, Any>(plugClass, socketClass)
}

private fun <SocketT, PlugT : SocketT> generatePlugin(
Expand Down Expand Up @@ -128,14 +108,18 @@ class PlugGenerator internal constructor(toSearches: List<File>, toLinkAgainst:
* Returns a Map from a plugin's name to its ATPLUG-INF content.
*
* @param toSearch a directory containing class files where we will look for plugin
* implementations
* implementations
* @param toLinkAgainst the classes that these plugins implementations need
* @return a map from component name to is ATPLUG-INF string content
*/
fun generate(toSearch: List<File>, toLinkAgainst: Set<File>): SortedMap<String, String> {
fun generate(
plugToSocket: Map<String, String>,
toSearch: List<File>,
toLinkAgainst: Set<File>
): SortedMap<String, String> {
return try {
val ext = PlugGeneratorJavaExecable(toSearch, toLinkAgainst)
val metadataGen = PlugGenerator(ext.toSearch, ext.toLinkAgainst)
val ext = PlugGeneratorJavaExecable(plugToSocket, toSearch, toLinkAgainst)
val metadataGen = PlugGenerator(plugToSocket, ext.toSearch, ext.toLinkAgainst)
// save our results, with no reference to the guts of what happened inside PluginMetadataGen
metadataGen.atplugInf
} catch (e: Exception) {
Expand Down
18 changes: 9 additions & 9 deletions atplug-plugin-gradle/src/main/java/com/diffplug/atplug/tooling/PlugGeneratorJavaExecable.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,21 +20,21 @@ import java.io.File
import java.util.*

/** [PlugGenerator.PlugGenerator] in a [JavaExecable] form. */
class PlugGeneratorJavaExecable(toSearch: List<File>?, toLinkAgainst: Set<File>?) : JavaExecable {
class PlugGeneratorJavaExecable(
plugToSocket: Map<String, String>,
toSearch: List<File>,
toLinkAgainst: Set<File>
) : JavaExecable {
// inputs
var toSearch: List<File>
var toLinkAgainst: Set<File>
var plugToSocket: Map<String, String> = LinkedHashMap(plugToSocket)
var toSearch: List<File> = ArrayList(toSearch)
var toLinkAgainst: Set<File> = LinkedHashSet(toLinkAgainst)

// outputs
@JvmField var atplugInf: SortedMap<String, String>? = null

init {
this.toSearch = ArrayList(toSearch)
this.toLinkAgainst = LinkedHashSet(toLinkAgainst)
}

override fun run() {
val metadataGen = PlugGenerator(toSearch, toLinkAgainst)
val metadataGen = PlugGenerator(plugToSocket, toSearch, toLinkAgainst)
atplugInf = metadataGen.atplugInf
}
}
17 changes: 8 additions & 9 deletions atplug-plugin-gradle/src/main/java/com/diffplug/atplug/tooling/PlugParser.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import org.objectweb.asm.Type
class PlugParser {
private var buffer = ByteArray(64 * 1024)

fun parse(file: File) {
fun parse(file: File): Pair<String, String>? {
val filelen = (file.length() + 1).toInt() // +1 prevents infinite loop
if (buffer.size < filelen) {
buffer = ByteArray(filelen)
Expand All @@ -42,18 +42,17 @@ class PlugParser {
}
}
val reader = ClassReader(buffer, 0, pos)
plugClassName = asmToJava(reader.className)
val plugClassName = asmToJava(reader.className)
this.plugClassName = plugClassName
socketClassName = null
// set socketClassName if there is an `@Plug`
reader.accept(
classVisitor, ClassReader.SKIP_FRAMES or ClassReader.SKIP_DEBUG or ClassReader.SKIP_CODE)
return socketClassName?.let { plugClassName to it }
}

fun hasPlug(): Boolean {
return socketClassName != null
}

var plugClassName: String? = null
var socketClassName: String? = null
private var plugClassName: String? = null
private var socketClassName: String? = null

private val classVisitor: ClassVisitor =
object : ClassVisitor(API) {
Expand All @@ -73,7 +72,7 @@ class PlugParser {
}

companion object {
fun asmToJava(className: String): String {
private fun asmToJava(className: String): String {
return className.replace("/", ".")
}

Expand Down
82 changes: 82 additions & 0 deletions atplug-plugin-gradle/src/main/java/com/diffplug/atplug/tooling/gradle/FindPlugsTask.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
package com.diffplug.atplug.tooling.gradle

import com.diffplug.atplug.tooling.PlugParser
import java.io.File
import org.gradle.api.DefaultTask
import org.gradle.api.file.ConfigurableFileCollection
import org.gradle.api.file.DirectoryProperty
import org.gradle.api.tasks.*
import org.gradle.work.*

/**
* Incrementally scans compiled classes for @Plug usage and writes discovered plug classes into an
* output directory.
*/
@CacheableTask
abstract class FindPlugsTask : DefaultTask() {
@get:CompileClasspath
@get:Incremental
@get:InputFiles
abstract val classesFolders: ConfigurableFileCollection

/** Directory where we will store discovered plugs. */
@get:OutputDirectory abstract val discoveredPlugsDir: DirectoryProperty

@TaskAction
fun findPlugs(inputChanges: InputChanges) {
// If not incremental, clear everything and rescan
if (!inputChanges.isIncremental) {
discoveredPlugsDir.get().asFile.deleteRecursively()
}

// Make sure our output directory exists
discoveredPlugsDir.get().asFile.mkdirs()

// For each changed file in classesFolders, determine if it has @Plug
val parser = PlugParser()
for (change in inputChanges.getFileChanges(classesFolders)) {
if (!change.file.name.endsWith(".class")) {
continue
}
when (change.changeType) {
ChangeType.REMOVED -> {
// Remove old discovered data for this file
removeOldMetadata(change.file)
}
ChangeType.ADDED,
ChangeType.MODIFIED -> {
parseAndWriteMetadata(parser, change.file)
}
}
}
}

private fun parseAndWriteMetadata(parser: PlugParser, classFile: File) {
val plugToSocket = parser.parse(classFile)
if (plugToSocket != null) {
// For example: write a single line containing the discovered plug FQN
val discoveredFile = discoveredPlugsDir.file(classFile.nameWithoutExtension).get().asFile
if (discoveredFile.exists()) {
val existing = discoveredFile.readText().split("|")
check(existing[0] == plugToSocket.first) {
"You need to rename one of these plugs because they have the same classfile name: ${existing[0]} and $plugToSocket"
}
} else {
discoveredFile.parentFile.mkdirs()
}
discoveredFile.writeText(plugToSocket.let { "${it.first}|${it.second}" })
} else {
// If previously discovered, remove it
removeOldMetadata(classFile)
}
}

private fun removeOldMetadata(classFile: File) {
// Remove any discovered file for the old .class
val possibleName = classFile.nameWithoutExtension
val discoveredFile = discoveredPlugsDir.file(possibleName).get().asFile
if (discoveredFile.exists()) {
discoveredFile.delete()
}
}
}
48 changes: 24 additions & 24 deletions atplug-plugin-gradle/src/main/java/com/diffplug/atplug/tooling/gradle/PlugGenerateTask.kt
View file
Open in desktop
Loading