Add support for Forge shipping with JarInJar options. (#10683)

These are optional libraries that are only loaded when requested by mods.
Includes new ReadOnlyInMemoryFileSystem to improve performance of JarInJar parsing.
This commit is contained in:
LexManos 2025-10-30 14:48:46 -07:00 committed by GitHub
parent 4376cd6b85
commit 6a615f1a4b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 1001 additions and 122 deletions

View file

@ -7,6 +7,7 @@ dependencies {
implementation 'org.ow2.asm:asm:9.8'
implementation 'org.ow2.asm:asm-tree:9.8'
implementation 'net.minecraftforge:srgutils:0.5.10'
implementation 'net.minecraftforge:JarJarMetadata:0.3.27'
implementation 'commons-io:commons-io:2.13.0'
implementation 'com.google.code.gson:gson:2.10.1'
implementation 'org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r'

View file

@ -0,0 +1,415 @@
/*
* Copyright (c) Forge Development LLC and contributors
* SPDX-License-Identifier: LGPL-2.1-only
*/
package net.minecraftforge.forge.tasks;
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import net.minecraftforge.jarjar.metadata.ContainedJarIdentifier;
import net.minecraftforge.jarjar.metadata.ContainedJarMetadata;
import net.minecraftforge.jarjar.metadata.ContainedVersion;
import net.minecraftforge.jarjar.metadata.Metadata;
import net.minecraftforge.jarjar.metadata.MetadataIOHandler;
import net.minecraftforge.jarjar.metadata.json.ArtifactVersionSerializer;
import net.minecraftforge.jarjar.metadata.json.ContainedJarIdentifierSerializer;
import net.minecraftforge.jarjar.metadata.json.ContainedJarMetadataSerializer;
import net.minecraftforge.jarjar.metadata.json.ContainedVersionSerializer;
import net.minecraftforge.jarjar.metadata.json.MetadataSerializer;
import net.minecraftforge.jarjar.metadata.json.VersionRangeSerializer;
import org.apache.maven.artifact.versioning.ArtifactVersion;
import org.apache.maven.artifact.versioning.DefaultArtifactVersion;
import org.apache.maven.artifact.versioning.InvalidVersionSpecificationException;
import org.apache.maven.artifact.versioning.VersionRange;
import org.codehaus.groovy.runtime.InvokerHelper;
import org.gradle.api.Action;
import org.gradle.api.DefaultTask;
import org.gradle.api.artifacts.ConfigurationContainer;
import org.gradle.api.artifacts.Dependency;
import org.gradle.api.artifacts.ExternalModuleDependency;
import org.gradle.api.artifacts.FileCollectionDependency;
import org.gradle.api.artifacts.MinimalExternalModuleDependency;
import org.gradle.api.artifacts.ModuleDependency;
import org.gradle.api.artifacts.ModuleIdentifier;
import org.gradle.api.artifacts.ModuleVersionIdentifier;
import org.gradle.api.artifacts.ResolvedArtifact;
import org.gradle.api.file.ProjectLayout;
import org.gradle.api.file.RegularFileProperty;
import org.gradle.api.provider.Provider;
import org.gradle.api.provider.SetProperty;
import org.gradle.api.tasks.Input;
import org.gradle.api.tasks.OutputFile;
import org.gradle.api.tasks.TaskAction;
import javax.inject.Inject;
import java.io.File;
import java.io.IOException;
import java.io.Serial;
import java.io.Serializable;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.zip.ZipFile;
// TODO SUPER SUPER SUPER UGLY, CLEAN UP IN FORGEDEV 7
@Deprecated(forRemoval = true) // Will be moved to JarJar plugin in ForgeDev 7
public abstract class JarJarMetadataOptions extends DefaultTask {
private static final Gson GSON = new GsonBuilder()
.registerTypeAdapter(VersionRange.class, new VersionRangeSerializer())
.registerTypeAdapter(ArtifactVersion.class, new ArtifactVersionSerializer())
.registerTypeAdapter(DefaultArtifactVersion.class, new ArtifactVersionSerializer())
.registerTypeAdapter(ContainedJarIdentifier.class, new ContainedJarIdentifierSerializer())
.registerTypeAdapter(ContainedJarMetadata.class, new ContainedJarMetadataSerializer())
.registerTypeAdapter(ContainedVersion.class, new ContainedVersionSerializer())
.registerTypeAdapter(Metadata.class, new MetadataSerializer())
.setPrettyPrinting()
.create();
protected abstract @Input SetProperty<ResolvedDependencyInfoImpl> getResolvedDependencies();
protected abstract @OutputFile RegularFileProperty getMetadataFile();
protected abstract @Inject ProjectLayout getLayout();
// NOTE: I'm not adding a non-provider version. please just use the version catalog entries for now.
public void add(Provider<MinimalExternalModuleDependency> dependency, Action<? super ResolvedDependencyInfo> action) {
this.getResolvedDependencies().add(dependency.map(d -> {
var ret = ResolvedDependencyInfoImpl.from(this.getProject().getConfigurations(), d);
action.execute(ret);
return ret;
}));
}
@Inject
public JarJarMetadataOptions() {
this.getMetadataFile().convention(this.getLayout().getBuildDirectory().file(this.getName() + "/options.json"));
}
@TaskAction
protected void exec() {
record ForgeLocaterOptions(String resource, String layer, String id, List<ContainedJarMetadata> deps, ContainedJarMetadata meta, boolean nested) { }
var resolved = this.getResolvedDependencies().get();
var jars = new ArrayList<ForgeLocaterOptions>(resolved.size());
for (var dependency : resolved) {
var deps = new ArrayList<ContainedJarMetadata>();
try (var zip = new ZipFile(dependency.artifact)) {
var entry = zip.getEntry("META-INF/jarjar/metadata.json");
if (entry != null) {
try (var stream = zip.getInputStream(entry)) {
var meta = MetadataIOHandler.fromStream(stream).orElse(null);
if (meta == null)
throw new IllegalStateException("Corrupt metadata.json in " + dependency.artifact.getAbsolutePath());
for (var dep : meta.jars())
deps.add(new ContainedJarMetadata(dep.identifier(), dep.version(), "", dep.isObfuscated()));
}
}
} catch (IOException e) {
throw new RuntimeException(e);
}
jars.add(new ForgeLocaterOptions(
dependency.resource,
dependency.layer,
dependency.identifier,
deps,
new ContainedJarMetadata(
new ContainedJarIdentifier(validateGroup(dependency), dependency.module.getName()),
new ContainedVersion(null, parseVersion(dependency)),
Objects.requireNonNull(dependency.path, "Dependency path is unspecified: " + dependency.asString),
false
),
dependency.nested
));
}
try {
record Meta(List<ForgeLocaterOptions> options){}
Files.writeString(
this.getMetadataFile().getAsFile().get().toPath(),
GSON.toJson(new Meta(jars), Meta.class)
);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private String validateGroup(ResolvedDependencyInfoImpl dependency) {
try {
return Objects.requireNonNull(dependency.module.getGroup());
} catch (NullPointerException e) {
throw new IllegalArgumentException("Module dependency has no group: " + dependency.asString, e);
}
}
private ArtifactVersion parseVersion(ResolvedDependencyInfoImpl resolved) {
try {
return VersionRange.createFromVersionSpec(Objects.requireNonNull(resolved.version)).getRecommendedVersion();
} catch (InvalidVersionSpecificationException e) {
throw new IllegalArgumentException("Version is invalid for: " + resolved.asString, e);
} catch (NullPointerException e) {
throw new IllegalArgumentException("Version is unspecified for: " + resolved.asString, e);
}
}
private VersionRange parseVersionRange(ResolvedDependencyInfoImpl dependency) {
if (dependency.hasManuallySpecifiedRange) {
try {
return VersionRange.createFromVersionSpec(dependency.versionRange);
} catch (InvalidVersionSpecificationException e) {
throw new IllegalArgumentException("Version is invalid for: " + dependency.asString, e);
}
} else {
try {
return VersionRange.createFromVersionSpec("[%s,)".formatted(Objects.requireNonNull(dependency.versionRange)));
} catch (InvalidVersionSpecificationException e) {
throw new IllegalArgumentException("Version range is invalid for: " + dependency.asString, e);
} catch (NullPointerException e) {
throw new IllegalArgumentException("Version is unspecified for: " + dependency.asString, e);
}
}
}
public interface ResolvedDependencyInfo {
void containedJarMetadata(Action<? super ContainedJarMetadataInfo> action);
void setResource(String resource);
void setLayer(String layer);
void setId(String identifier);
void setNested(boolean nested);
interface ContainedJarMetadataInfo {
void setGroup(String group);
void setName(String name);
void setPath(String version);
}
}
static final class ResolvedDependencyInfoImpl implements ResolvedDependencyInfo, ResolvedDependencyInfo.ContainedJarMetadataInfo, Serializable {
private static final @Serial long serialVersionUID = -7577318115877822993L;
final MinimalModuleVersionIdentifier module;
final String version;
String versionRange;
boolean hasManuallySpecifiedRange;
boolean nested;
final File artifact;
String path;
String resource;
String layer;
String identifier;
final String asString;
public ResolvedDependencyInfoImpl(MinimalModuleVersionIdentifier module, String version, File artifact, String asString) {
this.module = module;
this.version = version;
this.artifact = artifact;
this.asString = asString;
}
@Override
public void containedJarMetadata(Action<? super ContainedJarMetadataInfo> action) {
action.execute(this);
}
@Override
public void setGroup(String group) {
this.module.group = group;
}
@Override
public void setName(String name) {
this.module.name = name;
}
@Override
public void setPath(String path) {
this.path = path;
}
@Override
public void setResource(String resource) {
this.resource = resource;
}
@Override
public void setLayer(String layer) {
this.layer = layer;
}
@Override
public void setId(String identifier) {
this.identifier = identifier;
}
@Override
public void setNested(boolean nested) {
this.nested = nested;
}
static Set<File> getFiles(Set<ResolvedDependencyInfoImpl> resolvedDependencies) {
var ret = new HashSet<File>(resolvedDependencies.size());
for (var dependency : resolvedDependencies) {
ret.add(dependency.artifact);
}
return ret;
}
static ResolvedDependencyInfoImpl from(ConfigurationContainer configurations, Dependency dependency) {
var group = dependency.getGroup();
var name = dependency.getName();
var version = dependency.getVersion();
if (dependency instanceof FileCollectionDependency filesDependency) {
File artifact;
try {
artifact = filesDependency.getFiles().getSingleFile();
} catch (IllegalStateException e) {
// TODO fileCollectionDependencyIsNotSingleFile
throw e;
}
return new ResolvedDependencyInfoImpl(
new MinimalModuleVersionIdentifier(group, name, version),
version,
artifact,
filesDependency.toString()
);
} else if (dependency instanceof ModuleDependency moduleDependency) {
moduleDependency = moduleDependency.copy();
if (moduleDependency instanceof ExternalModuleDependency externalModuleDependency) {
externalModuleDependency.version(v -> v.strictly(version.toString()));
}
var detachedConfiguration = configurations.detachedConfiguration(moduleDependency);
detachedConfiguration.setTransitive(false);
ResolvedDependencyInfoImpl ret = null;
for (var artifact : detachedConfiguration.getResolvedConfiguration().getFirstLevelModuleDependencies().iterator().next().getModuleArtifacts()) {
var fileName = getFileName(artifact);
if (!fileName.endsWith(".jar"))
continue;
if (ret != null)
throw new IllegalArgumentException("Module dependency has too many Jar artifacts: " + moduleDependency);
ret = new ResolvedDependencyInfoImpl(
new MinimalModuleVersionIdentifier(group, name, artifact.getModuleVersion().getId().getVersion()),
version,
artifact.getFile(),
moduleDependency.toString()
);
}
if (ret == null)
throw new IllegalArgumentException("Module dependency has no Jar artifacts: " + moduleDependency);
return ret;
} else {
throw new IllegalArgumentException("Unsupported dependency type: " + dependency.getClass().getName() + " -- " + dependency);
}
}
private static String getFileName(ResolvedArtifact artifact) {
try {
return InvokerHelper.getProperty(artifact.getId(), "fileName").toString();
} catch (Throwable e) {
// NOTE: Why not just use this to begin with?
// ComponentArtifactIdentifier can have a getFileName() method, which doesn't necessarily resolve the file itself.
// This allows us to get the name of the file to be used without asking Gradle to download the file.
// So, if a file is not a JAR file, we can check the name without actually downloading it.
return artifact.getFile().getName();
}
}
@Override
public boolean equals(Object obj) {
if (obj == this) return true;
if (obj == null || obj.getClass() != this.getClass()) return false;
var that = (ResolvedDependencyInfoImpl) obj;
return Objects.equals(this.module, that.module) &&
Objects.equals(this.version, that.version) &&
Objects.equals(this.versionRange, that.versionRange) &&
Objects.equals(this.artifact, that.artifact);
}
@Override
public int hashCode() {
return Objects.hash(module, version, versionRange, artifact);
}
@Override
public String toString() {
return "ResolvedDependencyInfo[" +
"module=" + module + ", " +
"fixedVersion=" + version + ", " +
"versionRange=" + versionRange + ", " +
"artifact=" + artifact + ']';
}
static final class MinimalModuleVersionIdentifier implements ModuleIdentifier, ModuleVersionIdentifier {
private static final @Serial long serialVersionUID = -955346236759069739L;
private String group;
private String name;
private final String version;
@Inject
public MinimalModuleVersionIdentifier(String group, String name, String version) {
this.group = group;
this.name = name;
this.version = version;
}
@Override
public ModuleIdentifier getModule() {
return this;
}
@Override
public String getGroup() {
return this.group;
}
@Override
public String getName() {
return this.name;
}
@Override
public String getVersion() {
return this.version;
}
@Override
public boolean equals(Object obj) {
return this == obj || obj instanceof MinimalModuleVersionIdentifier o
&& Objects.equals(this.group, o.group)
&& Objects.equals(this.name, o.name)
&& Objects.equals(this.version, o.version);
}
@Override
public int hashCode() {
return Objects.hash(group, name, version);
}
@Override
public String toString() {
return "MinimalModuleVersionIdentifier[" +
"group=" + group + ", " +
"name=" + name + ", " +
"version=" + version + ']';
}
}
}
}

View file

@ -135,6 +135,7 @@ dependencies {
installer(libs.bundles.terminalconsoleappender)
installer(libs.mixin)
installer(libs.bundles.jarjar)
installer(libs.roimfs)
installer(project(':fmlcore'))
installer(project(':fmlloader'))

View file

@ -38,6 +38,10 @@ tasks.register('generateResources') {
dependsOn('writeManifest')
}
tasks.named('processResources') {
dependsOn(generateResources)
}
// Make sure out manifests get written before compiling the code, IDEA calls this task if you tell it to use the gradle build.
tasks.withType(JavaCompile).configureEach {
dependsOn 'generateResources'

View file

@ -36,6 +36,7 @@ dependencies {
implementation(libs.accesstransformers)
implementation(libs.terminalconsoleappender)
implementation(libs.jimfs)
implementation(libs.roimfs)
// Needed because we have a custom log4j plugin, and they removed package scanning and require a data file to be generated
implementation(libs.log4j.core)
@ -103,11 +104,22 @@ tasks.register('writeForgeVersionJson') {
}
}
tasks.register('jarJarOptionsJson', net.minecraftforge.forge.tasks.JarJarMetadataOptions) {
metadataFile = project.file('src/main/resources/jarjar_options.json')
// This is resolved too early by cpw.mods.modlauncher.TransformationServicesHandler.discoverServices(DiscoveryData) But eventually...
//add(libs.mixin, 'org/spongepowered/asm/mixin/Mixin.class')
}
tasks.named('generateResources').configure {
dependsOn('eclipseJdt')
dependsOn('eclipseJdtApt')
dependsOn('eclipseFactorypath')
dependsOn('writeForgeVersionJson')
dependsOn('jarJarOptionsJson')
}
tasks.named('sourcesJar') {
dependsOn('jarJarOptionsJson')
}
eclipse {

View file

@ -87,7 +87,7 @@ public abstract class AbstractModProvider implements IModProvider {
return new IModLocator.ModFileOrException(mod, null);
}
private static JarMetadata loadMetaFromJar(SecureJar jar, ModJarMetadata mjm) {
protected static JarMetadata loadMetaFromJar(SecureJar jar, ModJarMetadata mjm) {
var info = jar.moduleDataProvider().open(MODULE_INFO).orElse(null);
if (info != null) {
try {

View file

@ -74,7 +74,8 @@ public final class ClasspathLocator extends AbstractModProvider implements IModL
var ret = new ArrayList<ModFileOrException>();
for (var path : claimed) {
// Filter out anything found by the ServiceLoader
if (!ModDirTransformerDiscoverer.isServiceProvider(path))
if (!ModDirTransformerDiscoverer.isServiceProvider(path)
&& !JarInJarDependencyLocator.isOption(path))
ret.add(createMod(path));
}
return ret;
@ -107,7 +108,7 @@ public final class ClasspathLocator extends AbstractModProvider implements IModL
return Path.of(URI.create(str));
}
private static Path getPathFromResource(ClassLoader cl, String resource) {
static Path getPathFromResource(ClassLoader cl, String resource) {
var url = cl.getResource(resource);
if (url == null)
return null;

View file

@ -5,22 +5,32 @@
package net.minecraftforge.fml.loading.moddiscovery;
import com.google.common.collect.ImmutableMap;
import com.electronwill.nightconfig.core.UnmodifiableConfig;
import com.electronwill.nightconfig.core.file.FileConfig;
import com.mojang.logging.LogUtils;
import net.minecraftforge.fml.loading.EarlyLoadingException;
import net.minecraftforge.fml.loading.EarlyLoadingException.ExceptionData;
import net.minecraftforge.forgespi.language.IModInfo;
import net.minecraftforge.forgespi.locating.IDependencyLocator;
import net.minecraftforge.forgespi.locating.IModFile;
import net.minecraftforge.forgespi.locating.ModFileLoadingException;
import net.minecraftforge.forgespi.locating.IModFile.Type;
import net.minecraftforge.jarjar.metadata.ContainedJarMetadata;
import net.minecraftforge.jarjar.metadata.MetadataIOHandler;
import net.minecraftforge.jarjar.selection.JarSelector;
import org.apache.maven.artifact.versioning.ArtifactVersion;
import org.apache.maven.artifact.versioning.VersionRange;
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.jetbrains.annotations.ApiStatus;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.Marker;
import org.slf4j.MarkerFactory;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.lang.module.ModuleDescriptor;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.FileSystem;
import java.nio.file.FileSystems;
import java.nio.file.Files;
@ -28,14 +38,19 @@ import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.StringJoiner;
import java.util.TreeMap;
import java.util.jar.JarFile;
import java.util.jar.Manifest;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ApiStatus.Internal
public class JarInJarDependencyLocator extends AbstractModProvider implements IDependencyLocator {
private static final String ROIMFS = "roimfs";
private static final String COLOR_CODE = "\u00a7";
private static final String RESET = COLOR_CODE + "r";
private static final String YELLOW = COLOR_CODE + "e";
@ -43,32 +58,192 @@ public class JarInJarDependencyLocator extends AbstractModProvider implements ID
private static final String GREEN = COLOR_CODE + "2";
private static final Logger LOGGER = LogUtils.getLogger();
private static final Marker MARKER = MarkerFactory.getMarker("JAR-JAR"); // LogMarkers.SCAN;
private static final Map<Path, Option> OPTIONS = new HashMap<>();
private static volatile boolean optionsLoaded = false;
@Override
public String name() {
return "JarInJar";
}
private int fsID = 0;
private synchronized int nextId() {
return fsID++;
}
@Override
public List<IModFile> scanMods(Iterable<IModFile> loadedMods) {
final List<IModFile> sources = new ArrayList<>();
loadedMods.forEach(sources::add);
var mods = new ArrayList<IModFile>();
loadedMods.forEach(mods::add);
var dependenciesToLoad = JarSelector.detectAndSelect(
sources,
this::loadResourceFromModFile,
this::loadModFileFrom,
this::identifyMod,
this::exception
);
var selector = new Selector(mods);
if (dependenciesToLoad.isEmpty()) {
if (selector.entries.size() == mods.size()) {
LOGGER.info("No dependencies to load found. Skipping!");
return Collections.emptyList();
}
LOGGER.info("Found {} dependencies adding them to mods collection", dependenciesToLoad.size());
return dependenciesToLoad;
loadOptions();
// Ideally, the json would be in order so we only have to loop once. But its written this way just in case.
var seen = new HashSet<Option>();
var added = false;
do {
added = false;
for (var entry : OPTIONS.values()) {
if (seen.contains(entry) || !selector.isRequired(entry.meta.identifier()))
continue;
seen.add(entry);
entry.addTo(selector);
if (entry.deps != null && !entry.deps.isEmpty()) {
added = true;
for (var dep : entry.deps)
selector.addRequirement(dep);
}
}
} while (added);
var selected = selector.select();
if (LOGGER.isDebugEnabled()) {
var ids = new TreeMap<String, List<Entry>>();
for (var entry : selector.entries.values()) {
if (entry != FAILED && entry.coord != null)
ids.computeIfAbsent(entry.coord, id -> new ArrayList<>()).add(entry);
}
for (var entry : ids.entrySet()) {
LOGGER.info(MARKER, "JarJar Candidated for {}", entry.getKey());
Collections.sort(entry.getValue());
for (var option : entry.getValue())
LOGGER.info(MARKER, "\t{}{}", selected.contains(option) ? '*' : ' ', option);
}
}
if (selected.isEmpty()) {
LOGGER.info("No dependencies to load found. Skipping!");
return Collections.emptyList();
}
/* Since Java 11, the JRE decompresses all zip files to memory when getting a ByteChannel for them.
* JarJar only targets Minecraft 1.18+ which is Java17+ Which means we can exploit the fact that
* these are Disjointed file systems, and close all unselected libraries.
*
* https://github.com/openjdk/jdk11u/blob/25a9b9e5594e88c9e1e66f95c7ca45d5a9da8854/src/jdk.zipfs/share/classes/jdk/nio/zipfs/ZipFileSystem.java#L703
*
* Combined with the fact that the JarJarFileSystem has always simply called into FileSystems.newFileSystem for its wrapped targets.
* https://github.com/MinecraftForge/JarJar/blame/b39b2051439c81519d826d38898778bf3081356f/src/main/java/net/minecraftforge/jarjar/nio/pathfs/PathFileSystem.java#L38
*
* This effectively means that JarJar has ALWAYS been a wrapper around in-memory copies of every jar it scans.
*
* It also never closes these FileSystems, which potentially means they are leaked if the GC can't inherently GC them.
*
* Ideally we would extract these contained jars to a temp folder, so that we can use the nice optimizations Java has for ZipFile/JarFile.
* But in order to do that we would need to write the file to a place that could potentially have conflicts.
* To resolve those conflicts we should take a hint out of maven/gradle/minecraft's book and key the filepath based on a hash of the file being extracted.
* However, because that hash is no available in existing metadata we would need to re-calculate it.
*
* To do it in an efficient way we I needed to be able to share the single opened FileSystem with both the SecureJar implementation, and with our recursive search.
* So I use my new ReadOnlyInMemoryFileSystem which is specifically designed to use the same backing array for the JarInJar search as it does with the resulting dependency
* selected SecureJars.
*
* This means that if we wanted to we could rather efficiently calculate a hash for extracting the file. However, it also means that we don't have to extract to file to
* support ZipFileSystem caching properly. So as long as we are okay with not using the JRE's optimized ZipFile implementation, or we want to support legacy JarInJar metadata,
* using this efficient in memory file system is the best option performance wise. Extracting the files during launch without an efficient caching/unique identifier system
* would incur a lot of Disc IO which can be super slow depending on your setup.
*
* So, it is a stated goal of our future re-write to extract files into a cache directory to gain the benefits of JRE optimizations, using a FileSystem much simpler then
* the existing JarInJarFileSystem is good enough.
*/
// Lets close all unselected FileSystems
var wasSelected = new HashSet<>(selected);
for (var entry : selector.entries.values())
entry.cleanup(wasSelected.contains(entry));
LOGGER.info(MARKER, "Found {} dependencies adding them to mods collection", selected.size());
var ret = new ArrayList<IModFile>(selected.size());
// Now lets build the actual mod files
for (var entry : selected) {
try {
var root = entry.path;
var mod = createMod(root, false, entry.type.name());
if (mod.ex() != null)
throw fail("Failed to load JarInJar file " + entry, mod.ex());
ret.add(mod.file());
} catch (EarlyLoadingException e) {
throw e;
} catch (Throwable e) {
throw fail("Failed to load JarInJar file " + entry, e);
}
}
return ret;
}
private record Options(List<OptionMetadata> options) {}
private record OptionMetadata(String resource, Type layer, String id, List<ContainedJarMetadata> deps, ContainedJarMetadata meta, boolean nested) {}
private static void loadOptions() {
if (optionsLoaded)
return;
synchronized (OPTIONS) {
if (optionsLoaded)
return;
optionsLoaded = true;
var self = JarInJarDependencyLocator.class;
try (var stream = self.getModule().getResourceAsStream("/jarjar_options.json")) {
if (stream == null) {
LOGGER.error(MARKER, "Failed to find jarjar_options.json");
return;
}
var meta = MetadataIOHandler.getGson().fromJson(new InputStreamReader(stream), Options.class);
//var meta = MetadataIOHandler.fromStream(stream).orElse(null);
if (meta == null) {
LOGGER.error(MARKER, "Corrupted jarjar_options.json");
return;
}
for (var jar : meta.options()) {
Path path = ClasspathLocator.getPathFromResource(self.getClassLoader(), jar.resource());
if (path == null) {
LOGGER.error(MARKER, "Failed to find JarJar option for {}", jar.resource());
continue;
}
LOGGER.debug(MARKER, "Found JarJar Option: {}", path.toAbsolutePath());
if (jar.nested())
OPTIONS.put(path, new NestedOption(jar.id(), path, jar.layer(), jar.meta(), jar.deps()));
else
OPTIONS.put(path, new Option(jar.id(), path, jar.layer(), jar.meta(), jar.deps()));
}
} catch (Throwable e) {
LOGGER.error(MARKER, "Failed to read JarJar Options file", e);
}
}
}
static boolean isOption(Path path) {
loadOptions();
return OPTIONS.containsKey(path);
}
private static EarlyLoadingException fail(String message, @Nullable Throwable cause) {
LOGGER.error(MARKER, message);
return new EarlyLoadingException(message, cause, Collections.emptyList());
}
@Override
@ -76,89 +251,7 @@ public class JarInJarDependencyLocator extends AbstractModProvider implements ID
return IModFile.Type.GAMELIBRARY.name();
}
protected Optional<IModFile> loadModFileFrom(IModFile file, Path path) {
try {
final Path pathInModFile = file.findResource(path.toString());
final URI filePathUri = new URI("jij:" + (pathInModFile.toAbsolutePath().toUri().getRawSchemeSpecificPart())).normalize();
final Map<String, ?> outerFsArgs = ImmutableMap.of("packagePath", pathInModFile);
final FileSystem zipFS = FileSystems.newFileSystem(filePathUri, outerFsArgs);
final Path pathInFS = zipFS.getPath("/");
final IModFile.Type parentType = file.getType();
final String modType;
if (parentType == IModFile.Type.LIBRARY || parentType == IModFile.Type.LANGPROVIDER) {
modType = IModFile.Type.LIBRARY.name();
} else {
modType = IModFile.Type.GAMELIBRARY.name();
}
return Optional.of(createMod(pathInFS, false, modType).file());
} catch (Exception e) {
LOGGER.error("Failed to load mod file {} from {}", path, file.getFileName());
final RuntimeException exception = new ModFileLoadingException("Failed to load mod file " + file.getFileName());
exception.initCause(e);
throw exception;
}
}
protected EarlyLoadingException exception(Collection<JarSelector.ResolutionFailureInformation<IModFile>> failedDependencies) {
final List<EarlyLoadingException.ExceptionData> errors = failedDependencies.stream()
.filter(entry -> !entry.sources().isEmpty()) //Should never be the case, but just to be sure
.map(JarInJarDependencyLocator::buildExceptionData)
.toList();
return new EarlyLoadingException(failedDependencies.size() + " Dependency restrictions were not met.", null, errors);
}
@NotNull
private static EarlyLoadingException.ExceptionData buildExceptionData(JarSelector.ResolutionFailureInformation<IModFile> entry) {
return new EarlyLoadingException.ExceptionData(
getErrorTranslationKey(entry),
entry.identifier().group() + ":" + entry.identifier().artifact(),
entry.sources()
.stream()
.flatMap(JarInJarDependencyLocator::getModWithVersionRangeStream)
.map(JarInJarDependencyLocator::formatError)
.collect(Collectors.joining(", "))
);
}
@NotNull
private static String getErrorTranslationKey(JarSelector.ResolutionFailureInformation<IModFile> entry) {
return entry.failureReason() == JarSelector.FailureReason.VERSION_RESOLUTION_FAILED ?
"fml.dependencyloading.conflictingdependencies" :
"fml.dependencyloading.mismatchedcontaineddependencies";
}
@NotNull
private static Stream<ModWithVersionRange> getModWithVersionRangeStream(JarSelector.SourceWithRequestedVersionRange<IModFile> file) {
return file.sources()
.stream()
.map(IModFile::getModFileInfo)
.flatMap(modFileInfo -> modFileInfo.getMods().stream())
.map(modInfo -> new ModWithVersionRange(modInfo, file.requestedVersionRange(), file.includedVersion()));
}
protected Optional<InputStream> loadResourceFromModFile(IModFile modFile, Path path) {
try {
var pathInModFile = modFile.findResource(path.toString());
if (!Files.exists(pathInModFile)) {
LOGGER.debug("Failed to load resource {} from {}, it does not contain dependency information.", path, modFile.getFileName());
return Optional.empty();
}
return Optional.of(Files.newInputStream(pathInModFile));
} catch (Exception e) {
LOGGER.error("Failed to load resource {} from mod {}, cause {}", path, modFile.getFileName(), e);
return Optional.empty();
}
}
@NotNull
private static String formatError(ModWithVersionRange modWithVersionRange){
return YELLOW + modWithVersionRange.modInfo().getModId() + RESET + " - " +
RED + modWithVersionRange.versionRange().toString() + RESET + " - " +
GREEN + modWithVersionRange.artifactVersion().toString() + RESET;
}
protected String identifyMod(IModFile modFile) {
protected static String identifyMod(IModFile modFile) {
if (modFile.getModFileInfo() != null && !modFile.getModInfos().isEmpty())
return modFile.getModInfos().stream().map(IModInfo::getModId).collect(Collectors.joining());
@ -169,5 +262,355 @@ public class JarInJarDependencyLocator extends AbstractModProvider implements ID
return modFile.getFileName();
}
private record ModWithVersionRange(IModInfo modInfo, VersionRange versionRange, ArtifactVersion artifactVersion) {}
private static final Entry FAILED = new Entry("FAILED ENTRY", Path.of(""), Type.LIBRARY, null);
private static sealed class Entry implements Comparable<Entry> {
final String id;
final Path path;
final Type type;
final @Nullable String coord;
private Entry(String id, Path path, Type type, @Nullable String coord) {
this.id = id;
this.path = path;
this.type = type;
this.coord = coord;
}
void cleanup(boolean selected) {
}
@Override
public String toString() {
if (path.getFileName() == null)
return path.toUri().toString();
return path.getFileName().toString();
}
@Nullable
Path getResource(String path) {
return null;
}
@Override
public int compareTo(Entry o) {
return o.toString().compareTo(toString());
}
}
private static final class ModEntry extends Entry {
private final IModFile mod;
ModEntry(IModFile mod) {
super(identifyMod(mod), mod.getFilePath(), mod.getType(), null);
this.mod = mod;
}
@Override
@Nullable
Path getResource(String path) {
return mod.findResource(path);
}
}
private static sealed class Option extends Entry {
final ContainedJarMetadata meta;
final List<ContainedJarMetadata> deps;
private Option(String id, Path path, Type type, ContainedJarMetadata meta, List<ContainedJarMetadata> deps) {
super(id, path, type, meta.identifier().group() + ':' + meta.identifier().artifact());
this.meta = meta;
this.deps = deps;
}
public void addTo(Selector selector) {
selector.option(this, meta);
}
}
private static final class NestedOption extends Option {
private FileSystem zip;
private Path root;
private NestedOption(String id, Path path, Type type, ContainedJarMetadata meta, List<ContainedJarMetadata> deps) {
super(id, path, type, meta, deps);
}
@Override
public void addTo(Selector selector) {
super.addTo(selector);
selector.add(this);
}
@Override
void cleanup(boolean selected) {
if (this.zip != null) {
try {
zip.close();
} catch (IOException e) {
LOGGER.error(MARKER, "Failed to close unselected FileSystem {}", this, e);
}
}
}
@Nullable
Path getResource(String path) {
if (zip == null) {
try {
this.zip = FileSystems.newFileSystem(this.path);
this.root = zip.getRootDirectories().iterator().next();
} catch (IOException e) {
LOGGER.error(MARKER, "Failed to open FileSystem for option root {}", this, e);
return null;
}
}
var target = root.resolve(path);
if (Files.exists(target))
return target;
return null;
}
}
private static final class Nested extends Entry {
private final Entry parent;
private final String nestedPath;
private final Path zipPath;
Nested(String id, Path path, Type type, String coords, Entry parent, String nestedPath, Path zipPath) {
super(id, path, type, coords);
this.parent = parent;
this.nestedPath = nestedPath;
this.zipPath = zipPath;
}
@Override
void cleanup(boolean selected) {
if (selected)
return;
LOGGER.info(MARKER, "Closeing unselected FileSystem {}", this);
try {
zipPath.getFileSystem().close();
path.getFileSystem().close();
} catch (IOException e) {
LOGGER.error(MARKER, "Failed to close unselected FileSystem {}", this, e);
}
}
@Override
@Nullable
Path getResource(String path) {
return zipPath.resolve(path);
}
@Override
public String toString() {
return parent + "!/" + nestedPath;
}
}
private record Key(Entry parent, String path) {}
private final class Selector extends JarSelector<Entry> {
private final HashMap<Key, Entry> entries = new HashMap<>();
private final HashMap<Entry, HashMap<String, String>> children = new HashMap<>();
private Selector(Iterable<IModFile> mods) {
for (var mod : mods) {
var entry = new ModEntry(mod);
this.force(entry);
entries.put(new Key(entry, ""), entry);
}
}
@Override
public void option(Entry entry, ContainedJarMetadata meta) {
super.option(entry, meta);
entries.put(new Key(entry, ""), entry);
}
@Override
public void add(Entry entry) {
super.add(entry);
entries.put(new Key(entry, ""), entry);
}
@Override
@Nullable
protected InputStream getResource(Entry source, String path) {
try {
Path target = source.getResource(path);
if (target == null || !Files.exists(target)) {
LOGGER.debug(MARKER, "Failed to load resource {} from {}, it does not contain dependency information.", path, source);
return null;
}
// Read the metadata file and store identifiers so we can group them later.
if (JarSelector.CONTAINED_JARS_METADATA_PATH.equals(path)) {
try (var is = Files.newInputStream(target)) {
var meta = MetadataIOHandler.fromStream(is).orElse(null);
if (meta == null)
return Files.newInputStream(target);
var children = new HashMap<String, String>();
for (var child : meta.jars())
children.put(child.path().replace('\\', '/'), child.identifier().group() + ':' + child.identifier().artifact());
this.children.put(source, children);
}
}
return Files.newInputStream(target);
} catch (Exception e) {
LOGGER.error(MARKER, "Failed to load resource {} from {}, cause {}", path, source, e);
return null;
}
}
@Override
protected Entry getNested(Entry source, String path) {
var cleaned = path.replace('\\', '/');
var key = new Key(source, cleaned);
var entry = entries.get(key);
if (entry == null) {
entry = createNested(source, path, cleaned);
if (entry == null)
entry = FAILED;
entries.put(key, entry);
}
return entry == FAILED ? null : entry;
}
private Entry createNested(Entry source, String path, String cleaned) {
var targetPath = source.getResource(path);
if (targetPath == null || !Files.isRegularFile(targetPath))
return null;
Path roimfsPath = null;
Path zipPath = null;
try {
// Lets copy the uncomressed file to memory, and then create a ReadOnlyInMemorryFileSystem for it, this allows us to reference it by URI.
// Which is needed because a lot of things require navigation via URI.
// And ZipFileSystem only caches the FileSystem instance when accessed via URI.
byte[] data = Files.readAllBytes(targetPath);
String filename = targetPath.getFileName().toString();
URI uri = new URI(ROIMFS, "jar-jar-" + nextId() + "/" + filename, null);
var roimfs = FileSystems.newFileSystem(uri, Collections.singletonMap("data", data));
roimfsPath = roimfs.getPath(filename);
var zip = FileSystems.newFileSystem(new URI("jar:" + roimfsPath.toUri()), Collections.emptyMap());
zipPath = zip.getRootDirectories().iterator().next(); // Get first root, which on ZipFileSystem is the only root
} catch (IOException | URISyntaxException e) {
LOGGER.error(MARKER, "Failed to create FileSystem for nested jar {}", source, e);
return null;
}
var mf = new Manifest();
var mfPath = zipPath.resolve(JarFile.MANIFEST_NAME);
if (Files.exists(mfPath)) {
try (var is = Files.newInputStream(mfPath)) {
mf = new Manifest(is);
} catch (IOException e) {
LOGGER.error(MARKER, "Failed to read manifest from {}", source, e);
}
}
IModFile.Type type = null;
String id = null;
var tomlPath = zipPath.resolve(MODS_TOML);
if (Files.exists(tomlPath)) {
var cfg = FileConfig.builder(tomlPath).build();
cfg.load();
cfg.close();
if (cfg.getOrElse("mods", null) instanceof Collection collection) {
@SuppressWarnings("unchecked")
var mods = (Collection<UnmodifiableConfig>)collection;
if (!mods.isEmpty()) {
var first = mods.iterator().next();
var modId = first.getOrElse("modId", null);
if (modId instanceof String str)
id = str;
}
}
var value = mf.getMainAttributes().getValue(ModFile.TYPE);
if (value != null)
type = IModFile.Type.valueOf(value);
else
type = IModFile.Type.MOD;
} else {
var value = mf.getMainAttributes().getValue(ModFile.TYPE);
if (value != null)
type = IModFile.Type.valueOf(value);
else {
// No mods.toml, no entry value, so decide based on the container's type
if (source.type == IModFile.Type.LIBRARY || source.type == IModFile.Type.LANGPROVIDER)
type = IModFile.Type.LIBRARY;
else
type = IModFile.Type.GAMELIBRARY;
}
}
if (id == null) {
var modulePath = zipPath.resolve(MODULE_INFO);
if (Files.exists(modulePath)) {
try (var is = Files.newInputStream(modulePath)) {
var module = ModuleDescriptor.read(is);
id = module.name();
} catch (IOException e) {
LOGGER.error(MARKER, "Failed to read module-info.class from {}", source, e);
}
}
}
if (id == null)
id = mf.getMainAttributes().getValue("Automatic-Module-Name");
if (id == null)
id = Path.of(path).getFileName().toString();
String coords = null;
var child = this.children.get(source);
if (child != null)
coords = child.get(cleaned);
return new Nested(id, roimfsPath, type, coords, source, cleaned, zipPath);
}
@Override
protected String getIdentifier(Entry entry) {
return entry.id;
}
@Override
protected Throwable getFailureException(Collection<ResolutionFailureInformation<Entry>> failures) {
var errors = new ArrayList<ExceptionData>();
for (var failure : failures) {
var message = failure.failureReason() == JarSelector.FailureReason.VERSION_RESOLUTION_FAILED ? "fml.dependencyloading.conflictingdependencies" : "fml.dependencyloading.mismatchedcontaineddependencies";
var joiner = new StringJoiner(", ");
for (var source : failure.sources()) {
String paths = null;
if (source.sources().size() == 0) {
paths = "[No Sources]";
} else if (source.sources().size() == 1) {
var entry = source.sources().iterator().next();
paths = entry.toString();
} else {
var j = new StringJoiner(", ", "[", "]");
for (var entry : source.sources())
j.add(entry.toString());
paths = j.toString();
}
var error = YELLOW + paths + RESET + " - " +
RED + source.requestedVersionRange() + RESET + " - " +
GREEN + source.includedVersion() + RESET;
joiner.add(error);
}
errors.add(new ExceptionData(message, failure.identifier().group() + ":" + failure.identifier().artifact(), joiner.toString()));
}
return new EarlyLoadingException(failures.size() + " Dependency restrictions were not met.", null, errors);
}
}
}

View file

@ -111,7 +111,7 @@ public final class ModDiscoverer {
}
LOGGER.debug(LogMarkers.SCAN, "Locator {} found {} valid mod files", locator, locatedFiles.size());
handleLocatedFiles(loadedFiles, locatedFiles);
handleLocatedFiles(loadedFiles, locatedFiles, locator);
} catch (InvalidModFileException imfe) {
// We don't generally expect this exception, since it should come from the candidates stream above and be handled in the Locator, but just in case.
LOGGER.error(LogMarkers.SCAN, "Locator {} found an invalid mod file {}", locator, imfe.getBrokenFile(), imfe);
@ -148,11 +148,7 @@ public final class ModDiscoverer {
final List<IModFile> locatedMods = List.copyOf(loadedFiles);
var locatedFiles = locator.scanMods(locatedMods);
if (locatedFiles.stream().anyMatch(file -> !(file instanceof ModFile))) {
LOGGER.error(LogMarkers.SCAN, "A dependency locator returned a file which is not a ModFile instance!. They will be skipped!");
}
handleLocatedFiles(loadedFiles, locatedFiles);
handleLocatedFiles(loadedFiles, locatedFiles, locator);
}
catch (EarlyLoadingException exception) {
LOGGER.error(LogMarkers.SCAN, "Failed to load dependencies with locator {}", locator, exception);
@ -185,11 +181,14 @@ public final class ModDiscoverer {
return validator;
}
private static void handleLocatedFiles(final List<ModFile> loadedFiles, final List<IModFile> locatedFiles) {
var locatedModFiles = locatedFiles.stream().filter(ModFile.class::isInstance).map(ModFile.class::cast).toList();
for (IModFile mf : locatedModFiles) {
LOGGER.info(LogMarkers.SCAN, "Found mod file {} of type {} with provider {}", mf.getFileName(), mf.getType(), mf.getProvider());
private static void handleLocatedFiles(final List<ModFile> loadedFiles, final List<IModFile> locatedFiles, final Object locator) {
for (IModFile mf : locatedFiles) {
if (mf instanceof ModFile modFile) {
LOGGER.info(LogMarkers.SCAN, "Found mod file {} of type {} with provider {}", mf.getFileName(), mf.getType(), mf.getProvider());
loadedFiles.add(modFile);
} else {
LOGGER.error(LogMarkers.SCAN, "Skipping mod file {} found by {}, as it was not a ModFile instance!", mf.getFileName(), locator);
}
}
loadedFiles.addAll(locatedModFiles);
}
}

View file

@ -0,0 +1,3 @@
{
"options": []
}

View file

@ -10,7 +10,6 @@ import cpw.mods.modlauncher.api.ITransformationService;
import cpw.mods.modlauncher.api.ITransformer;
import org.jetbrains.annotations.NotNull;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;

View file

@ -27,7 +27,7 @@ dependencyResolutionManagement {
library('forgespi', 'net.minecraftforge:forgespi:7.1.5') // Needs modlauncher
library('modlauncher', 'net.minecraftforge:modlauncher:10.2.4') // Needs securemodules
library('securemodules', 'net.minecraftforge:securemodules:2.2.21') // Needs unsafe
library('securemodules', 'net.minecraftforge:securemodules:2.2.23') // Needs unsafe
library('unsafe', 'net.minecraftforge:unsafe:0.9.2')
library('accesstransformers', 'net.minecraftforge:accesstransformers:8.2.2')
library('coremods', 'net.minecraftforge:coremods:5.2.6')
@ -36,6 +36,7 @@ dependencyResolutionManagement {
library('eventbus-validator', 'net.minecraftforge:eventbus-validator:7.0-beta.12')
library('mergetool-api', 'net.minecraftforge:mergetool-api:1.0')
library('roimfs', 'net.minecraftforge:roimfs:1.0.0') // ReadOnlyInMemoryFileSystem - Used for JarInJar extraction at runtime
library('mixin', 'org.spongepowered:mixin:0.8.7')
library('jetbrains-annotations', 'org.jetbrains:annotations:24.1.0') // for ApiStatus annotations
library('jspecify', 'org.jspecify:jspecify:1.0.0') // for nullability annotations
@ -76,7 +77,7 @@ dependencyResolutionManagement {
bundle('night-config', ['night-config-toml', 'night-config-core'])
// Jar in Jar FileSystem
version('jarjar', '0.3.26')
version('jarjar', '0.4.2')
library('jarjar-fs', 'net.minecraftforge', 'JarJarFileSystems').versionRef('jarjar')
library('jarjar-meta', 'net.minecraftforge', 'JarJarMetadata' ).versionRef('jarjar')
library('jarjar-selector', 'net.minecraftforge', 'JarJarSelector' ).versionRef('jarjar')