From 6a615f1a4b1053fe3d8c39a8ca1f802e403dfcff Mon Sep 17 00:00:00 2001 From: LexManos Date: Thu, 30 Oct 2025 14:48:46 -0700 Subject: [PATCH] Add support for Forge shipping with JarInJar options. (#10683) These are optional libraries that are only loaded when requested by mods. Includes new ReadOnlyInMemoryFileSystem to improve performance of JarInJar parsing. --- buildSrc/build.gradle | 1 + .../forge/tasks/JarJarMetadataOptions.java | 415 +++++++++++ build_forge.gradle | 1 + build_shared.gradle | 4 + fmlloader/build.gradle | 12 + .../moddiscovery/AbstractModProvider.java | 2 +- .../moddiscovery/ClasspathLocator.java | 5 +- .../JarInJarDependencyLocator.java | 653 +++++++++++++++--- .../loading/moddiscovery/ModDiscoverer.java | 21 +- .../src/main/resources/jarjar_options.json | 3 + .../transformers/ForgeModTransformers.java | 1 - settings.gradle | 5 +- 12 files changed, 1001 insertions(+), 122 deletions(-) create mode 100644 buildSrc/src/main/groovy/net/minecraftforge/forge/tasks/JarJarMetadataOptions.java create mode 100644 fmlloader/src/main/resources/jarjar_options.json diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 6206c624e5..0f87c9c66b 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -7,6 +7,7 @@ dependencies { implementation 'org.ow2.asm:asm:9.8' implementation 'org.ow2.asm:asm-tree:9.8' implementation 'net.minecraftforge:srgutils:0.5.10' + implementation 'net.minecraftforge:JarJarMetadata:0.3.27' implementation 'commons-io:commons-io:2.13.0' implementation 'com.google.code.gson:gson:2.10.1' implementation 'org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r' diff --git a/buildSrc/src/main/groovy/net/minecraftforge/forge/tasks/JarJarMetadataOptions.java b/buildSrc/src/main/groovy/net/minecraftforge/forge/tasks/JarJarMetadataOptions.java new file mode 100644 index 0000000000..8837a29032 --- /dev/null +++ b/buildSrc/src/main/groovy/net/minecraftforge/forge/tasks/JarJarMetadataOptions.java @@ -0,0 +1,415 @@ +/* + * Copyright (c) Forge Development LLC and contributors + * SPDX-License-Identifier: LGPL-2.1-only + */ + +package net.minecraftforge.forge.tasks; + +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import net.minecraftforge.jarjar.metadata.ContainedJarIdentifier; +import net.minecraftforge.jarjar.metadata.ContainedJarMetadata; +import net.minecraftforge.jarjar.metadata.ContainedVersion; +import net.minecraftforge.jarjar.metadata.Metadata; +import net.minecraftforge.jarjar.metadata.MetadataIOHandler; +import net.minecraftforge.jarjar.metadata.json.ArtifactVersionSerializer; +import net.minecraftforge.jarjar.metadata.json.ContainedJarIdentifierSerializer; +import net.minecraftforge.jarjar.metadata.json.ContainedJarMetadataSerializer; +import net.minecraftforge.jarjar.metadata.json.ContainedVersionSerializer; +import net.minecraftforge.jarjar.metadata.json.MetadataSerializer; +import net.minecraftforge.jarjar.metadata.json.VersionRangeSerializer; +import org.apache.maven.artifact.versioning.ArtifactVersion; +import org.apache.maven.artifact.versioning.DefaultArtifactVersion; +import org.apache.maven.artifact.versioning.InvalidVersionSpecificationException; +import org.apache.maven.artifact.versioning.VersionRange; +import org.codehaus.groovy.runtime.InvokerHelper; +import org.gradle.api.Action; +import org.gradle.api.DefaultTask; +import org.gradle.api.artifacts.ConfigurationContainer; +import org.gradle.api.artifacts.Dependency; +import org.gradle.api.artifacts.ExternalModuleDependency; +import org.gradle.api.artifacts.FileCollectionDependency; +import org.gradle.api.artifacts.MinimalExternalModuleDependency; +import org.gradle.api.artifacts.ModuleDependency; +import org.gradle.api.artifacts.ModuleIdentifier; +import org.gradle.api.artifacts.ModuleVersionIdentifier; +import org.gradle.api.artifacts.ResolvedArtifact; +import org.gradle.api.file.ProjectLayout; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.provider.Provider; +import org.gradle.api.provider.SetProperty; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.OutputFile; +import org.gradle.api.tasks.TaskAction; + +import javax.inject.Inject; +import java.io.File; +import java.io.IOException; +import java.io.Serial; +import java.io.Serializable; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.zip.ZipFile; + +// TODO SUPER SUPER SUPER UGLY, CLEAN UP IN FORGEDEV 7 +@Deprecated(forRemoval = true) // Will be moved to JarJar plugin in ForgeDev 7 +public abstract class JarJarMetadataOptions extends DefaultTask { + private static final Gson GSON = new GsonBuilder() + .registerTypeAdapter(VersionRange.class, new VersionRangeSerializer()) + .registerTypeAdapter(ArtifactVersion.class, new ArtifactVersionSerializer()) + .registerTypeAdapter(DefaultArtifactVersion.class, new ArtifactVersionSerializer()) + .registerTypeAdapter(ContainedJarIdentifier.class, new ContainedJarIdentifierSerializer()) + .registerTypeAdapter(ContainedJarMetadata.class, new ContainedJarMetadataSerializer()) + .registerTypeAdapter(ContainedVersion.class, new ContainedVersionSerializer()) + .registerTypeAdapter(Metadata.class, new MetadataSerializer()) + .setPrettyPrinting() + .create(); + + protected abstract @Input SetProperty getResolvedDependencies(); + + protected abstract @OutputFile RegularFileProperty getMetadataFile(); + + protected abstract @Inject ProjectLayout getLayout(); + + // NOTE: I'm not adding a non-provider version. please just use the version catalog entries for now. + public void add(Provider dependency, Action action) { + this.getResolvedDependencies().add(dependency.map(d -> { + var ret = ResolvedDependencyInfoImpl.from(this.getProject().getConfigurations(), d); + action.execute(ret); + return ret; + })); + } + + @Inject + public JarJarMetadataOptions() { + this.getMetadataFile().convention(this.getLayout().getBuildDirectory().file(this.getName() + "/options.json")); + } + + @TaskAction + protected void exec() { + record ForgeLocaterOptions(String resource, String layer, String id, List deps, ContainedJarMetadata meta, boolean nested) { } + + var resolved = this.getResolvedDependencies().get(); + var jars = new ArrayList(resolved.size()); + for (var dependency : resolved) { + var deps = new ArrayList(); + try (var zip = new ZipFile(dependency.artifact)) { + var entry = zip.getEntry("META-INF/jarjar/metadata.json"); + if (entry != null) { + try (var stream = zip.getInputStream(entry)) { + var meta = MetadataIOHandler.fromStream(stream).orElse(null); + if (meta == null) + throw new IllegalStateException("Corrupt metadata.json in " + dependency.artifact.getAbsolutePath()); + for (var dep : meta.jars()) + deps.add(new ContainedJarMetadata(dep.identifier(), dep.version(), "", dep.isObfuscated())); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + jars.add(new ForgeLocaterOptions( + dependency.resource, + dependency.layer, + dependency.identifier, + deps, + new ContainedJarMetadata( + new ContainedJarIdentifier(validateGroup(dependency), dependency.module.getName()), + new ContainedVersion(null, parseVersion(dependency)), + Objects.requireNonNull(dependency.path, "Dependency path is unspecified: " + dependency.asString), + false + ), + dependency.nested + )); + } + + try { + record Meta(List options){} + + Files.writeString( + this.getMetadataFile().getAsFile().get().toPath(), + GSON.toJson(new Meta(jars), Meta.class) + ); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private String validateGroup(ResolvedDependencyInfoImpl dependency) { + try { + return Objects.requireNonNull(dependency.module.getGroup()); + } catch (NullPointerException e) { + throw new IllegalArgumentException("Module dependency has no group: " + dependency.asString, e); + } + } + + private ArtifactVersion parseVersion(ResolvedDependencyInfoImpl resolved) { + try { + return VersionRange.createFromVersionSpec(Objects.requireNonNull(resolved.version)).getRecommendedVersion(); + } catch (InvalidVersionSpecificationException e) { + throw new IllegalArgumentException("Version is invalid for: " + resolved.asString, e); + } catch (NullPointerException e) { + throw new IllegalArgumentException("Version is unspecified for: " + resolved.asString, e); + } + } + + private VersionRange parseVersionRange(ResolvedDependencyInfoImpl dependency) { + if (dependency.hasManuallySpecifiedRange) { + try { + return VersionRange.createFromVersionSpec(dependency.versionRange); + } catch (InvalidVersionSpecificationException e) { + throw new IllegalArgumentException("Version is invalid for: " + dependency.asString, e); + } + } else { + try { + return VersionRange.createFromVersionSpec("[%s,)".formatted(Objects.requireNonNull(dependency.versionRange))); + } catch (InvalidVersionSpecificationException e) { + throw new IllegalArgumentException("Version range is invalid for: " + dependency.asString, e); + } catch (NullPointerException e) { + throw new IllegalArgumentException("Version is unspecified for: " + dependency.asString, e); + } + } + } + + public interface ResolvedDependencyInfo { + void containedJarMetadata(Action action); + + void setResource(String resource); + + void setLayer(String layer); + + void setId(String identifier); + + void setNested(boolean nested); + + interface ContainedJarMetadataInfo { + void setGroup(String group); + + void setName(String name); + + void setPath(String version); + } + } + + static final class ResolvedDependencyInfoImpl implements ResolvedDependencyInfo, ResolvedDependencyInfo.ContainedJarMetadataInfo, Serializable { + private static final @Serial long serialVersionUID = -7577318115877822993L; + + final MinimalModuleVersionIdentifier module; + final String version; + String versionRange; + boolean hasManuallySpecifiedRange; + boolean nested; + final File artifact; + String path; + String resource; + String layer; + String identifier; + final String asString; + + public ResolvedDependencyInfoImpl(MinimalModuleVersionIdentifier module, String version, File artifact, String asString) { + this.module = module; + this.version = version; + this.artifact = artifact; + this.asString = asString; + } + + @Override + public void containedJarMetadata(Action action) { + action.execute(this); + } + + @Override + public void setGroup(String group) { + this.module.group = group; + } + + @Override + public void setName(String name) { + this.module.name = name; + } + + @Override + public void setPath(String path) { + this.path = path; + } + + @Override + public void setResource(String resource) { + this.resource = resource; + } + + @Override + public void setLayer(String layer) { + this.layer = layer; + } + + @Override + public void setId(String identifier) { + this.identifier = identifier; + } + + @Override + public void setNested(boolean nested) { + this.nested = nested; + } + + static Set getFiles(Set resolvedDependencies) { + var ret = new HashSet(resolvedDependencies.size()); + for (var dependency : resolvedDependencies) { + ret.add(dependency.artifact); + } + return ret; + } + + static ResolvedDependencyInfoImpl from(ConfigurationContainer configurations, Dependency dependency) { + var group = dependency.getGroup(); + var name = dependency.getName(); + var version = dependency.getVersion(); + + if (dependency instanceof FileCollectionDependency filesDependency) { + File artifact; + try { + artifact = filesDependency.getFiles().getSingleFile(); + } catch (IllegalStateException e) { + // TODO fileCollectionDependencyIsNotSingleFile + throw e; + } + + return new ResolvedDependencyInfoImpl( + new MinimalModuleVersionIdentifier(group, name, version), + version, + artifact, + filesDependency.toString() + ); + } else if (dependency instanceof ModuleDependency moduleDependency) { + moduleDependency = moduleDependency.copy(); + if (moduleDependency instanceof ExternalModuleDependency externalModuleDependency) { + externalModuleDependency.version(v -> v.strictly(version.toString())); + } + + var detachedConfiguration = configurations.detachedConfiguration(moduleDependency); + detachedConfiguration.setTransitive(false); + + ResolvedDependencyInfoImpl ret = null; + for (var artifact : detachedConfiguration.getResolvedConfiguration().getFirstLevelModuleDependencies().iterator().next().getModuleArtifacts()) { + var fileName = getFileName(artifact); + if (!fileName.endsWith(".jar")) + continue; + + if (ret != null) + throw new IllegalArgumentException("Module dependency has too many Jar artifacts: " + moduleDependency); + + ret = new ResolvedDependencyInfoImpl( + new MinimalModuleVersionIdentifier(group, name, artifact.getModuleVersion().getId().getVersion()), + version, + artifact.getFile(), + moduleDependency.toString() + ); + } + if (ret == null) + throw new IllegalArgumentException("Module dependency has no Jar artifacts: " + moduleDependency); + + return ret; + } else { + throw new IllegalArgumentException("Unsupported dependency type: " + dependency.getClass().getName() + " -- " + dependency); + } + } + + private static String getFileName(ResolvedArtifact artifact) { + try { + return InvokerHelper.getProperty(artifact.getId(), "fileName").toString(); + } catch (Throwable e) { + // NOTE: Why not just use this to begin with? + // ComponentArtifactIdentifier can have a getFileName() method, which doesn't necessarily resolve the file itself. + // This allows us to get the name of the file to be used without asking Gradle to download the file. + // So, if a file is not a JAR file, we can check the name without actually downloading it. + return artifact.getFile().getName(); + } + } + + @Override + public boolean equals(Object obj) { + if (obj == this) return true; + if (obj == null || obj.getClass() != this.getClass()) return false; + var that = (ResolvedDependencyInfoImpl) obj; + return Objects.equals(this.module, that.module) && + Objects.equals(this.version, that.version) && + Objects.equals(this.versionRange, that.versionRange) && + Objects.equals(this.artifact, that.artifact); + } + + @Override + public int hashCode() { + return Objects.hash(module, version, versionRange, artifact); + } + + @Override + public String toString() { + return "ResolvedDependencyInfo[" + + "module=" + module + ", " + + "fixedVersion=" + version + ", " + + "versionRange=" + versionRange + ", " + + "artifact=" + artifact + ']'; + } + + static final class MinimalModuleVersionIdentifier implements ModuleIdentifier, ModuleVersionIdentifier { + private static final @Serial long serialVersionUID = -955346236759069739L; + + private String group; + private String name; + private final String version; + + @Inject + public MinimalModuleVersionIdentifier(String group, String name, String version) { + this.group = group; + this.name = name; + this.version = version; + } + + @Override + public ModuleIdentifier getModule() { + return this; + } + + @Override + public String getGroup() { + return this.group; + } + + @Override + public String getName() { + return this.name; + } + + @Override + public String getVersion() { + return this.version; + } + + @Override + public boolean equals(Object obj) { + return this == obj || obj instanceof MinimalModuleVersionIdentifier o + && Objects.equals(this.group, o.group) + && Objects.equals(this.name, o.name) + && Objects.equals(this.version, o.version); + } + + @Override + public int hashCode() { + return Objects.hash(group, name, version); + } + + @Override + public String toString() { + return "MinimalModuleVersionIdentifier[" + + "group=" + group + ", " + + "name=" + name + ", " + + "version=" + version + ']'; + } + } + } +} diff --git a/build_forge.gradle b/build_forge.gradle index 4be10a27a6..6a68e14b79 100644 --- a/build_forge.gradle +++ b/build_forge.gradle @@ -135,6 +135,7 @@ dependencies { installer(libs.bundles.terminalconsoleappender) installer(libs.mixin) installer(libs.bundles.jarjar) + installer(libs.roimfs) installer(project(':fmlcore')) installer(project(':fmlloader')) diff --git a/build_shared.gradle b/build_shared.gradle index 36074208c9..b988510649 100644 --- a/build_shared.gradle +++ b/build_shared.gradle @@ -38,6 +38,10 @@ tasks.register('generateResources') { dependsOn('writeManifest') } +tasks.named('processResources') { + dependsOn(generateResources) +} + // Make sure out manifests get written before compiling the code, IDEA calls this task if you tell it to use the gradle build. tasks.withType(JavaCompile).configureEach { dependsOn 'generateResources' diff --git a/fmlloader/build.gradle b/fmlloader/build.gradle index bc3badd86c..f122226cec 100644 --- a/fmlloader/build.gradle +++ b/fmlloader/build.gradle @@ -36,6 +36,7 @@ dependencies { implementation(libs.accesstransformers) implementation(libs.terminalconsoleappender) implementation(libs.jimfs) + implementation(libs.roimfs) // Needed because we have a custom log4j plugin, and they removed package scanning and require a data file to be generated implementation(libs.log4j.core) @@ -103,11 +104,22 @@ tasks.register('writeForgeVersionJson') { } } +tasks.register('jarJarOptionsJson', net.minecraftforge.forge.tasks.JarJarMetadataOptions) { + metadataFile = project.file('src/main/resources/jarjar_options.json') + // This is resolved too early by cpw.mods.modlauncher.TransformationServicesHandler.discoverServices(DiscoveryData) But eventually... + //add(libs.mixin, 'org/spongepowered/asm/mixin/Mixin.class') +} + tasks.named('generateResources').configure { dependsOn('eclipseJdt') dependsOn('eclipseJdtApt') dependsOn('eclipseFactorypath') dependsOn('writeForgeVersionJson') + dependsOn('jarJarOptionsJson') +} + +tasks.named('sourcesJar') { + dependsOn('jarJarOptionsJson') } eclipse { diff --git a/fmlloader/src/main/java/net/minecraftforge/fml/loading/moddiscovery/AbstractModProvider.java b/fmlloader/src/main/java/net/minecraftforge/fml/loading/moddiscovery/AbstractModProvider.java index a8b77b1c2c..53e9af9c47 100644 --- a/fmlloader/src/main/java/net/minecraftforge/fml/loading/moddiscovery/AbstractModProvider.java +++ b/fmlloader/src/main/java/net/minecraftforge/fml/loading/moddiscovery/AbstractModProvider.java @@ -87,7 +87,7 @@ public abstract class AbstractModProvider implements IModProvider { return new IModLocator.ModFileOrException(mod, null); } - private static JarMetadata loadMetaFromJar(SecureJar jar, ModJarMetadata mjm) { + protected static JarMetadata loadMetaFromJar(SecureJar jar, ModJarMetadata mjm) { var info = jar.moduleDataProvider().open(MODULE_INFO).orElse(null); if (info != null) { try { diff --git a/fmlloader/src/main/java/net/minecraftforge/fml/loading/moddiscovery/ClasspathLocator.java b/fmlloader/src/main/java/net/minecraftforge/fml/loading/moddiscovery/ClasspathLocator.java index 7d8ba90442..cba72c99e7 100644 --- a/fmlloader/src/main/java/net/minecraftforge/fml/loading/moddiscovery/ClasspathLocator.java +++ b/fmlloader/src/main/java/net/minecraftforge/fml/loading/moddiscovery/ClasspathLocator.java @@ -74,7 +74,8 @@ public final class ClasspathLocator extends AbstractModProvider implements IModL var ret = new ArrayList(); for (var path : claimed) { // Filter out anything found by the ServiceLoader - if (!ModDirTransformerDiscoverer.isServiceProvider(path)) + if (!ModDirTransformerDiscoverer.isServiceProvider(path) + && !JarInJarDependencyLocator.isOption(path)) ret.add(createMod(path)); } return ret; @@ -107,7 +108,7 @@ public final class ClasspathLocator extends AbstractModProvider implements IModL return Path.of(URI.create(str)); } - private static Path getPathFromResource(ClassLoader cl, String resource) { + static Path getPathFromResource(ClassLoader cl, String resource) { var url = cl.getResource(resource); if (url == null) return null; diff --git a/fmlloader/src/main/java/net/minecraftforge/fml/loading/moddiscovery/JarInJarDependencyLocator.java b/fmlloader/src/main/java/net/minecraftforge/fml/loading/moddiscovery/JarInJarDependencyLocator.java index 557b77c89a..8dc21e0e52 100644 --- a/fmlloader/src/main/java/net/minecraftforge/fml/loading/moddiscovery/JarInJarDependencyLocator.java +++ b/fmlloader/src/main/java/net/minecraftforge/fml/loading/moddiscovery/JarInJarDependencyLocator.java @@ -5,22 +5,32 @@ package net.minecraftforge.fml.loading.moddiscovery; -import com.google.common.collect.ImmutableMap; +import com.electronwill.nightconfig.core.UnmodifiableConfig; +import com.electronwill.nightconfig.core.file.FileConfig; import com.mojang.logging.LogUtils; + import net.minecraftforge.fml.loading.EarlyLoadingException; +import net.minecraftforge.fml.loading.EarlyLoadingException.ExceptionData; import net.minecraftforge.forgespi.language.IModInfo; import net.minecraftforge.forgespi.locating.IDependencyLocator; import net.minecraftforge.forgespi.locating.IModFile; -import net.minecraftforge.forgespi.locating.ModFileLoadingException; +import net.minecraftforge.forgespi.locating.IModFile.Type; +import net.minecraftforge.jarjar.metadata.ContainedJarMetadata; +import net.minecraftforge.jarjar.metadata.MetadataIOHandler; import net.minecraftforge.jarjar.selection.JarSelector; -import org.apache.maven.artifact.versioning.ArtifactVersion; -import org.apache.maven.artifact.versioning.VersionRange; -import org.jetbrains.annotations.ApiStatus; -import org.jetbrains.annotations.NotNull; -import org.slf4j.Logger; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.Marker; +import org.slf4j.MarkerFactory; + +import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; +import java.lang.module.ModuleDescriptor; import java.net.URI; +import java.net.URISyntaxException; import java.nio.file.FileSystem; import java.nio.file.FileSystems; import java.nio.file.Files; @@ -28,14 +38,19 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; +import java.util.StringJoiner; +import java.util.TreeMap; +import java.util.jar.JarFile; +import java.util.jar.Manifest; import java.util.stream.Collectors; -import java.util.stream.Stream; @ApiStatus.Internal public class JarInJarDependencyLocator extends AbstractModProvider implements IDependencyLocator { + private static final String ROIMFS = "roimfs"; private static final String COLOR_CODE = "\u00a7"; private static final String RESET = COLOR_CODE + "r"; private static final String YELLOW = COLOR_CODE + "e"; @@ -43,32 +58,192 @@ public class JarInJarDependencyLocator extends AbstractModProvider implements ID private static final String GREEN = COLOR_CODE + "2"; private static final Logger LOGGER = LogUtils.getLogger(); + private static final Marker MARKER = MarkerFactory.getMarker("JAR-JAR"); // LogMarkers.SCAN; + + private static final Map OPTIONS = new HashMap<>(); + private static volatile boolean optionsLoaded = false; @Override public String name() { return "JarInJar"; } + private int fsID = 0; + private synchronized int nextId() { + return fsID++; + } + @Override public List scanMods(Iterable loadedMods) { - final List sources = new ArrayList<>(); - loadedMods.forEach(sources::add); + var mods = new ArrayList(); + loadedMods.forEach(mods::add); - var dependenciesToLoad = JarSelector.detectAndSelect( - sources, - this::loadResourceFromModFile, - this::loadModFileFrom, - this::identifyMod, - this::exception - ); + var selector = new Selector(mods); - if (dependenciesToLoad.isEmpty()) { + if (selector.entries.size() == mods.size()) { LOGGER.info("No dependencies to load found. Skipping!"); return Collections.emptyList(); } - LOGGER.info("Found {} dependencies adding them to mods collection", dependenciesToLoad.size()); - return dependenciesToLoad; + loadOptions(); + + // Ideally, the json would be in order so we only have to loop once. But its written this way just in case. + var seen = new HashSet