From d31023db249a3e61a5b8f1f0bacd2e91467d31d8 Mon Sep 17 00:00:00 2001 From: sipkab <10866741+Sipkab@users.noreply.github.com> Date: Thu, 9 Jan 2020 16:11:20 +0100 Subject: [PATCH 1/5] Minor typo fixes --- .../main/saker/java/compiler/api/classpath/ClassPathEntry.java | 2 +- .../jdk9/tasks/javac/MultiRelease9JarClasspathTaskTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/main/saker/java/compiler/api/classpath/ClassPathEntry.java b/api/src/main/saker/java/compiler/api/classpath/ClassPathEntry.java index b809711..47f535d 100644 --- a/api/src/main/saker/java/compiler/api/classpath/ClassPathEntry.java +++ b/api/src/main/saker/java/compiler/api/classpath/ClassPathEntry.java @@ -120,7 +120,7 @@ public default StructuredTaskResult getDocumentationAttachment() { * In general, if the implementation version key of a classpath changes, the dependent code needs to be re-executed * or recompiled. *

- * The implementation verison key should include all aspects of the class files available in the (non-transitive) + * The implementation version key should include all aspects of the class files available in the (non-transitive) * classpath. * * @return The implementation version key or null if none. diff --git a/main/src/test/testing/saker/java/compiler/tests/jdk9/tasks/javac/MultiRelease9JarClasspathTaskTest.java b/main/src/test/testing/saker/java/compiler/tests/jdk9/tasks/javac/MultiRelease9JarClasspathTaskTest.java index 2c0c44f..8ae4cfe 100644 --- a/main/src/test/testing/saker/java/compiler/tests/jdk9/tasks/javac/MultiRelease9JarClasspathTaskTest.java +++ b/main/src/test/testing/saker/java/compiler/tests/jdk9/tasks/javac/MultiRelease9JarClasspathTaskTest.java @@ -32,7 +32,7 @@ * This test is disabled for now, as the javac implementation doesn't work. *

* In order to have multi-release JARs on the classpath, the hidden --multi-release option needs to - * be added with the verison number as its argument to javac. + * be added with the version number as its argument to javac. *

* Then, it will use JarFileSystem to access the JAR files. However, it fails to properly list the * versioned entries, therefore the version 9 of the given class will not be enumerated. From a91fb784651cd98e9c91f1cc9f33863867ba991a Mon Sep 17 00:00:00 2001 From: sipkab <10866741+Sipkab@users.noreply.github.com> Date: Thu, 9 Jan 2020 18:34:07 +0100 Subject: [PATCH 2/5] Cache annotation processor classpaths in repository storage dir The input classpaths from where the saker.java.processor task loads the annotation processors are cached in the storage directory associated with the saker.java.compiler package. See issue #1 --- .../java/compiler/impl/JavaTaskUtils.java | 189 +----- .../ClassLoaderProcessorCreator.java | 249 ++++---- .../ProcessorCreatorTaskFactory.java | 566 +++++++++++++++++- 3 files changed, 679 insertions(+), 325 deletions(-) diff --git a/impl/src/main/saker/java/compiler/impl/JavaTaskUtils.java b/impl/src/main/saker/java/compiler/impl/JavaTaskUtils.java index b9fcd7f..4a9bef3 100644 --- a/impl/src/main/saker/java/compiler/impl/JavaTaskUtils.java +++ b/impl/src/main/saker/java/compiler/impl/JavaTaskUtils.java @@ -16,7 +16,6 @@ package saker.java.compiler.impl; import java.io.Externalizable; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; @@ -24,10 +23,8 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; -import java.util.Map; import java.util.Map.Entry; import java.util.NavigableMap; import java.util.NavigableSet; @@ -36,27 +33,15 @@ import java.util.TreeSet; import java.util.function.Function; -import saker.build.file.SakerDirectory; -import saker.build.file.SakerFile; import saker.build.file.content.ContentDescriptor; -import saker.build.file.content.DirectoryContentDescriptor; -import saker.build.file.content.MultiPathContentDescriptor; -import saker.build.file.content.SerializableContentDescriptor; import saker.build.file.path.SakerPath; import saker.build.file.provider.FileEntry; import saker.build.file.provider.LocalFileProvider; -import saker.build.file.provider.SakerPathFiles; import saker.build.runtime.execution.ExecutionContext; import saker.build.runtime.execution.ExecutionProperty; -import saker.build.runtime.execution.SakerLog; -import saker.build.task.TaskContext; -import saker.build.task.TaskDependencyFuture; -import saker.build.task.TaskExecutionUtilities; import saker.build.task.dependencies.CommonTaskOutputChangeDetector; -import saker.build.task.dependencies.FileCollectionStrategy; import saker.build.task.dependencies.TaskOutputChangeDetector; import saker.build.task.identifier.TaskIdentifier; -import saker.build.task.utils.dependencies.RecursiveIgnoreCaseExtensionFileCollectionStrategy; import saker.build.thirdparty.org.objectweb.asm.ClassReader; import saker.build.thirdparty.org.objectweb.asm.ClassVisitor; import saker.build.thirdparty.org.objectweb.asm.ModuleVisitor; @@ -67,28 +52,11 @@ import saker.build.thirdparty.saker.util.io.ByteArrayRegion; import saker.build.thirdparty.saker.util.io.FileUtils; import saker.build.thirdparty.saker.util.io.SerialUtils; -import saker.java.compiler.api.classpath.ClassPathEntry; -import saker.java.compiler.api.classpath.ClassPathReference; -import saker.java.compiler.api.classpath.ClassPathVisitor; -import saker.java.compiler.api.classpath.CompilationClassPath; -import saker.java.compiler.api.classpath.FileClassPath; -import saker.java.compiler.api.classpath.JavaClassPath; -import saker.java.compiler.api.classpath.JavaClassPathBuilder; -import saker.java.compiler.api.classpath.SDKClassPath; import saker.java.compiler.api.compile.JavaCompilationWorkerTaskIdentifier; -import saker.java.compiler.api.compile.JavaCompilerWorkerTaskOutput; import saker.java.compiler.api.compile.SakerJavaCompilerUtils; import saker.java.compiler.api.option.JavaAddExports; -import saker.java.compiler.impl.JavaTaskUtils.LocalDirectoryClassFilesExecutionProperty.PropertyValue; import saker.java.compiler.impl.compile.InternalJavaCompilerOutput; import saker.java.compiler.impl.compile.util.LocalPathFileContentDescriptorExecutionProperty; -import saker.sdk.support.api.SDKPathReference; -import saker.sdk.support.api.SDKReference; -import saker.sdk.support.api.SDKSupportUtils; -import saker.std.api.file.location.ExecutionFileLocation; -import saker.std.api.file.location.FileLocation; -import saker.std.api.file.location.FileLocationVisitor; -import saker.std.api.file.location.LocalFileLocation; public class JavaTaskUtils { public static final String EXTENSION_CLASSFILE = "class"; @@ -217,162 +185,7 @@ public static Collection toAddExportsCommandLineStrings(JavaAddExports a return SakerJavaCompilerUtils.toAddExportsCommandLineStrings(addexports); } - /** - * @return The content descriptors may be null. - */ - public static Map collectFileLocationsWithImplementationDependencyReporting( - TaskContext taskcontext, JavaClassPath classpath, Object tag, Map sdks, - Function classpathentryfilelocationhandler) throws IOException { - if (classpath == null) { - return Collections.emptyMap(); - } - Map result = new LinkedHashMap<>(); - classpath.accept(new ClassPathVisitor() { - private Set handledWorkerTaskIds = new HashSet<>(); - - @Override - public void visit(ClassPathReference classpath) { - Collection entries = classpath.getEntries(); - if (ObjectUtils.isNullOrEmpty(entries)) { - SakerLog.warning().println("No class path entries found for: " + classpath); - return; - } - for (ClassPathEntry entry : entries) { - if (entry == null) { - SakerLog.warning().println("Class path entry is null for: " + classpath); - continue; - } - FileLocation filelocation = classpathentryfilelocationhandler.apply(entry); - if (filelocation == null) { - SakerLog.warning().println("No class path file location for: " + entry); - continue; - } - handleFileLocation(filelocation); - - Collection additionalclasspaths = entry - .getAdditionalClassPathReferences(); - if (!ObjectUtils.isNullOrEmpty(additionalclasspaths)) { - JavaClassPathBuilder additionalcpbuilder = JavaClassPathBuilder.newBuilder(); - for (ClassPathReference additionalcp : additionalclasspaths) { - additionalcpbuilder.addClassPath(additionalcp); - } - JavaClassPath additionalcp = additionalcpbuilder.build(); - additionalcp.accept(this); - } - } - } - - @Override - public void visit(CompilationClassPath classpath) { - JavaCompilationWorkerTaskIdentifier workertaskid = classpath.getCompilationWorkerTaskIdentifier(); - if (!handledWorkerTaskIds.add(workertaskid)) { - //don't get the task result to not install another dependency - return; - } - TaskDependencyFuture depresult = taskcontext.getTaskDependencyFuture(workertaskid); - JavaCompilerWorkerTaskOutput output = (JavaCompilerWorkerTaskOutput) depresult.getFinished(); - SakerPath classdirpath = output.getClassDirectory(); - ExecutionFileLocation filelocation = ExecutionFileLocation.create(classdirpath); - JavaClassPath outputcp = output.getClassPath(); - - Object implversionkey = output.getImplementationVersionKey(); - if (implversionkey != null) { - depresult.setTaskOutputChangeDetector(SakerJavaCompilerUtils - .getCompilerOutputImplementationVersionKeyTaskOutputChangeDetector(implversionkey)); - depresult.setTaskOutputChangeDetector( - SakerJavaCompilerUtils.getCompilerOutputClassPathTaskOutputChangeDetector(outputcp)); - result.put(filelocation, new SerializableContentDescriptor(implversionkey)); - } else { - SakerDirectory classesdir = taskcontext.getTaskUtilities().resolveDirectoryAtPath(classdirpath); - if (classesdir == null) { - throw ObjectUtils.sneakyThrow( - new FileNotFoundException("Compilation class directory not found: " + classesdir)); - } - - FileCollectionStrategy classfileadditiondep = RecursiveIgnoreCaseExtensionFileCollectionStrategy - .create(classdirpath, "." + EXTENSION_CLASSFILE); - NavigableMap classfiles = taskcontext.getTaskUtilities() - .collectFilesReportInputFileAndAdditionDependency(tag, classfileadditiondep); - - NavigableMap contentmap = SakerPathFiles.toFileContentMap(classfiles); - result.put(filelocation, new MultiPathContentDescriptor(contentmap)); - } - if (outputcp != null) { - outputcp.accept(this); - } - } - - @Override - public void visit(FileClassPath classpath) { - FileLocation location = classpath.getFileLocation(); - handleFileLocation(location); - } - - @Override - public void visit(SDKClassPath classpath) { - SDKPathReference sdkpathref = classpath.getSDKPathReference(); - SakerPath path = SDKSupportUtils.getSDKPathReferencePath(sdkpathref, sdks); - LocalFileLocation fileloc = LocalFileLocation.create(path); - result.put(fileloc, null); - } - - private ContentDescriptor handleExecutionFileLocation(SakerPath path, SakerFile cpfile) { - if (cpfile instanceof SakerDirectory) { - FileCollectionStrategy classfileadditiondep = RecursiveIgnoreCaseExtensionFileCollectionStrategy - .create(path, "." + EXTENSION_CLASSFILE); - NavigableMap classfiles = taskcontext.getTaskUtilities() - .collectFilesReportInputFileAndAdditionDependency(tag, classfileadditiondep); - return new MultiPathContentDescriptor(SakerPathFiles.toFileContentMap(classfiles)); - } - taskcontext.getTaskUtilities().reportInputFileDependency(tag, cpfile); - return cpfile.getContentDescriptor(); - } - - private void handleFileLocation(FileLocation location) { - if (result.containsKey(location)) { - return; - } - ContentDescriptor[] cdres = { null }; - location.accept(new FileLocationVisitor() { - @Override - public void visit(ExecutionFileLocation loc) { - SakerPath path = loc.getPath(); - SakerFile cpfile = taskcontext.getTaskUtilities().resolveAtPath(path); - if (cpfile == null) { - throw ObjectUtils - .sneakyThrow(new FileNotFoundException("Class path file not found: " + path)); - } - cdres[0] = handleExecutionFileLocation(path, cpfile); - } - - @Override - public void visit(LocalFileLocation loc) { - SakerPath path = loc.getLocalPath(); - TaskExecutionUtilities taskutils = taskcontext.getTaskUtilities(); - ContentDescriptor cd = taskutils.getReportExecutionDependency( - new LocalPathFileContentDescriptorExecutionProperty(path)); - if (cd == null) { - throw ObjectUtils - .sneakyThrow(new FileNotFoundException("Class path local file not found: " + path)); - } - - if (DirectoryContentDescriptor.INSTANCE.equals(cd)) { - //the class path denotes a directory - //add the dependencies on the class files - - PropertyValue pval = taskutils - .getReportExecutionDependency(new LocalDirectoryClassFilesExecutionProperty(path)); - cdres[0] = new MultiPathContentDescriptor(pval.getContents()); - } else { - cdres[0] = cd; - } - } - }); - result.put(location, cdres[0]); - } - }); - return result; - } + public static class LocalDirectoryClassFilesExecutionProperty implements ExecutionProperty, Externalizable { diff --git a/impl/src/main/saker/java/compiler/impl/processor/ClassLoaderProcessorCreator.java b/impl/src/main/saker/java/compiler/impl/processor/ClassLoaderProcessorCreator.java index 3316854..0d0cd18 100644 --- a/impl/src/main/saker/java/compiler/impl/processor/ClassLoaderProcessorCreator.java +++ b/impl/src/main/saker/java/compiler/impl/processor/ClassLoaderProcessorCreator.java @@ -16,60 +16,38 @@ package saker.java.compiler.impl.processor; import java.io.Externalizable; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; -import java.lang.reflect.InvocationTargetException; -import java.nio.file.Path; -import java.util.ArrayList; +import java.lang.reflect.Constructor; import java.util.Arrays; -import java.util.Collection; +import java.util.LinkedHashSet; import java.util.Objects; import java.util.Set; -import java.util.UUID; -import java.util.function.Supplier; +import java.util.jar.JarFile; import javax.annotation.processing.Processor; -import saker.build.file.SakerDirectory; -import saker.build.file.SakerFile; -import saker.build.file.path.SakerPath; -import saker.build.file.provider.FileEntry; import saker.build.file.provider.LocalFileProvider; -import saker.build.file.provider.SakerPathFiles; -import saker.build.thirdparty.saker.util.ObjectUtils; -import saker.build.thirdparty.saker.util.ReflectUtils; +import saker.build.runtime.environment.SakerEnvironment; import saker.build.thirdparty.saker.util.classloader.ClassLoaderDataFinder; import saker.build.thirdparty.saker.util.classloader.ClassLoaderUtil; import saker.build.thirdparty.saker.util.classloader.JarClassLoaderDataFinder; import saker.build.thirdparty.saker.util.classloader.MultiClassLoader; import saker.build.thirdparty.saker.util.classloader.MultiDataClassLoader; -import saker.build.thirdparty.saker.util.classloader.PathClassLoaderDataFinder; -import saker.build.thirdparty.saker.util.io.ResourceCloser; +import saker.build.thirdparty.saker.util.io.JarFileUtils; import saker.build.thirdparty.saker.util.io.SerialUtils; +import saker.build.util.cache.CacheKey; import saker.java.compiler.api.processing.SakerProcessingEnvironment; import saker.java.compiler.api.processor.ProcessorCreationContext; import saker.java.compiler.api.processor.ProcessorCreator; -import saker.java.compiler.impl.compile.handler.ProcessorCreationContextImpl; -import saker.std.api.file.location.ExecutionFileLocation; -import saker.std.api.file.location.FileLocation; -import saker.std.api.file.location.FileLocationVisitor; import saker.std.api.file.location.LocalFileLocation; public final class ClassLoaderProcessorCreator implements ProcessorCreator, Externalizable { private static final long serialVersionUID = 1L; - private transient String className; - private transient Set fileLocations; - - /** - * An UUID that identifies this processor version instance. It is newly generated every time the processor creating - * task is run. - */ - private UUID changedUUID; - - private transient volatile Supplier supplier; + private String className; + private Set classPathJarFileLocations; /** * For {@link Externalizable}. @@ -77,13 +55,12 @@ public final class ClassLoaderProcessorCreator implements ProcessorCreator, Exte public ClassLoaderProcessorCreator() { } - public ClassLoaderProcessorCreator(String className, Set fileLocations) { + public ClassLoaderProcessorCreator(String className, Set classPathJarFileLocations) { Objects.requireNonNull(className, "className"); - Objects.requireNonNull(fileLocations, "fileLocations"); + Objects.requireNonNull(classPathJarFileLocations, "classPathJarFileLocations"); this.className = className; - this.fileLocations = fileLocations; - this.changedUUID = UUID.randomUUID(); + this.classPathJarFileLocations = classPathJarFileLocations; } @Override @@ -93,116 +70,31 @@ public String getName() { @Override public Processor create(ProcessorCreationContext creationcontext) throws Exception { - Supplier supp = this.supplier; - if (supp != null) { - return supp.get(); - } - //TODO REWORK this implementation to cache the classpath in the bundle storage and use cached data to properly close the JARs - ProcessorCreationContextImpl contextimpl = (ProcessorCreationContextImpl) creationcontext; - synchronized (this) { - if (this.supplier == null) { - Collection datafinders = new ArrayList<>(fileLocations.size()); - try (ResourceCloser closer = new ResourceCloser()) { - for (FileLocation filelocation : fileLocations) { - //XXX delay the jar opening until a class or file is accessed from it - filelocation.accept(new FileLocationVisitor() { - @Override - public void visit(ExecutionFileLocation loc) { - SakerFile file = SakerPathFiles.resolveAtPath(contextimpl.getExecutionContext(), null, - loc.getPath()); - try { - if (file == null) { - throw new FileNotFoundException( - "Processor class path file not found: " + loc.getPath()); - } - Path mirrorpath = contextimpl.mirror(file); - ClassLoaderDataFinder clfinder; - if (file instanceof SakerDirectory) { - clfinder = new PathClassLoaderDataFinder(mirrorpath); - } else { - clfinder = new JarClassLoaderDataFinder(mirrorpath); - } - datafinders.add(clfinder); - } catch (IOException e) { - throw ObjectUtils.sneakyThrow(e); - } - } - - @Override - public void visit(LocalFileLocation loc) { - SakerPath path = loc.getLocalPath(); - try { - Path realpath = LocalFileProvider.toRealPath(path); - FileEntry attrs = LocalFileProvider.getInstance().getFileAttributes(realpath); - ClassLoaderDataFinder clfinder; - if (attrs.isDirectory()) { - clfinder = new PathClassLoaderDataFinder(realpath); - } else { - clfinder = new JarClassLoaderDataFinder(realpath); - } - datafinders.add(clfinder); - } catch (IOException e) { - throw ObjectUtils.sneakyThrow(e); - } - } - }); - } - //TODO use environment caching for the class loader and the files and jars - // if someone wants to unpack to the file we use as classloader, then some invalidation should go through - - //the processor parent classloader is the API bundle classloader - // so they can access the API classes of the incremental compiler - //the platform classloader is also added, so the javac classes are also available to the processor - ClassLoader parentcl = SakerProcessingEnvironment.class.getClassLoader(); - try { - ClassLoader cl = new MultiDataClassLoader( - MultiClassLoader.create( - Arrays.asList(parentcl, ClassLoaderUtil.getPlatformClassLoaderParent())), - datafinders); - Class procclass = Class.forName(className, false, cl) - .asSubclass(Processor.class); - //XXX reify exceptions - supplier = () -> { - try { - return ReflectUtils.newInstance(procclass); - } catch (InstantiationException | IllegalAccessException | IllegalArgumentException - | InvocationTargetException | NoSuchMethodException | SecurityException e) { - throw new IllegalArgumentException("Failed to instantiate Processor: " + className, e); - } - }; - } catch (ClassNotFoundException e) { - throw new ClassNotFoundException( - "Processor class not found: " + className + " in " + datafinders, e); - } catch (ClassCastException e) { - throw new IllegalArgumentException( - "Class doesn't implement " + Processor.class.getName() + ": " + className, e); - } - closer.clearWithoutClosing(); - } - } - } - return this.supplier.get(); + SakerEnvironment environment = creationcontext.getEnvironment(); + Constructor constructor = environment + .getCachedData(new ProcessorConstructorCacheKey(environment, className, + classPathJarFileLocations)); + return constructor.newInstance(); } @Override public void writeExternal(ObjectOutput out) throws IOException { out.writeUTF(className); - SerialUtils.writeExternalCollection(out, fileLocations); - out.writeObject(changedUUID); + SerialUtils.writeExternalCollection(out, classPathJarFileLocations); } @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { className = in.readUTF(); - fileLocations = SerialUtils.readExternalImmutableLinkedHashSet(in); - changedUUID = (UUID) in.readObject(); + classPathJarFileLocations = SerialUtils.readExternalImmutableLinkedHashSet(in); } @Override public int hashCode() { final int prime = 31; int result = 1; - result = prime * result + ((changedUUID == null) ? 0 : changedUUID.hashCode()); + result = prime * result + ((className == null) ? 0 : className.hashCode()); + result = prime * result + ((classPathJarFileLocations == null) ? 0 : classPathJarFileLocations.hashCode()); return result; } @@ -215,12 +107,107 @@ public boolean equals(Object obj) { if (getClass() != obj.getClass()) return false; ClassLoaderProcessorCreator other = (ClassLoaderProcessorCreator) obj; - if (changedUUID == null) { - if (other.changedUUID != null) + if (className == null) { + if (other.className != null) + return false; + } else if (!className.equals(other.className)) + return false; + if (classPathJarFileLocations == null) { + if (other.classPathJarFileLocations != null) return false; - } else if (!changedUUID.equals(other.changedUUID)) + } else if (!classPathJarFileLocations.equals(other.classPathJarFileLocations)) return false; return true; } + @Override + public String toString() { + return getClass().getSimpleName() + "[className=" + className + ", classPathJarFileLocations=" + + classPathJarFileLocations + "]"; + } + + private static class JarClassLoaderCacheKey implements CacheKey { + private LocalFileLocation fileLocation; + + public JarClassLoaderCacheKey(LocalFileLocation fileLocation) { + this.fileLocation = fileLocation; + } + + @Override + public JarFile allocate() throws Exception { + return JarFileUtils.createMultiReleaseJarFile(LocalFileProvider.toRealPath(fileLocation.getLocalPath())); + } + + @Override + public void close(JarClassLoaderDataFinder data, JarFile resource) throws Exception { + resource.close(); + } + + @Override + public JarClassLoaderDataFinder generate(JarFile resource) throws Exception { + return new JarClassLoaderDataFinder(resource); + } + + @Override + public long getExpiry() { + // 5 min + return 5 * 60 * 1000; + } + + @Override + public boolean validate(JarClassLoaderDataFinder data, JarFile resource) { + return true; + } + } + + private static class ProcessorConstructorCacheKey implements CacheKey, Object> { + private static final ClassLoader PROCESSOR_PARENT_CLASSLOADER = MultiClassLoader.create(Arrays.asList( + SakerProcessingEnvironment.class.getClassLoader(), ClassLoaderUtil.getPlatformClassLoaderParent())); + + private SakerEnvironment environment; + private String className; + private Set cacheKeys; + + public ProcessorConstructorCacheKey(SakerEnvironment environment, String className, + Set files) { + this.environment = environment; + this.className = className; + this.cacheKeys = new LinkedHashSet<>(); + for (LocalFileLocation f : files) { + cacheKeys.add(new JarClassLoaderCacheKey(f)); + } + } + + @Override + public Object allocate() throws Exception { + return new Object(); + } + + @Override + public void close(Constructor data, Object resource) throws Exception { + } + + @Override + public Constructor generate(Object resource) throws Exception { + Set datafinders = new LinkedHashSet<>(); + for (JarClassLoaderCacheKey ck : cacheKeys) { + JarClassLoaderDataFinder jarcldf = environment.getCachedData(ck); + datafinders.add(jarcldf); + } + MultiDataClassLoader cl = new MultiDataClassLoader(PROCESSOR_PARENT_CLASSLOADER, datafinders); + return Class.forName(className, false, cl).asSubclass(Processor.class).getConstructor(); + } + + @Override + public long getExpiry() { + // 5 min + return 5 * 60 * 1000; + } + + @Override + public boolean validate(Constructor data, Object resource) { + return true; + } + + } } \ No newline at end of file diff --git a/impl/src/main/saker/java/compiler/impl/processor/ProcessorCreatorTaskFactory.java b/impl/src/main/saker/java/compiler/impl/processor/ProcessorCreatorTaskFactory.java index 50d78b4..9264886 100644 --- a/impl/src/main/saker/java/compiler/impl/processor/ProcessorCreatorTaskFactory.java +++ b/impl/src/main/saker/java/compiler/impl/processor/ProcessorCreatorTaskFactory.java @@ -16,37 +16,106 @@ package saker.java.compiler.impl.processor; import java.io.Externalizable; +import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; import java.io.ObjectInput; import java.io.ObjectOutput; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.FileTime; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Collection; +import java.util.Collections; +import java.util.ConcurrentModificationException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.Map; +import java.util.Map.Entry; import java.util.NavigableMap; +import java.util.NavigableSet; import java.util.Set; import java.util.TreeMap; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; +import saker.build.file.DirectoryVisitPredicate; +import saker.build.file.SakerDirectory; +import saker.build.file.SakerFile; +import saker.build.file.content.ContentDescriptor; +import saker.build.file.content.DirectoryContentDescriptor; +import saker.build.file.content.MultiPathContentDescriptor; +import saker.build.file.content.SerializableContentDescriptor; +import saker.build.file.path.SakerPath; +import saker.build.file.provider.FileEntry; +import saker.build.file.provider.FileHashResult; +import saker.build.file.provider.LocalFileProvider; +import saker.build.file.provider.SakerPathFiles; import saker.build.runtime.execution.ExecutionContext; +import saker.build.runtime.execution.ExecutionProperty; +import saker.build.runtime.execution.SakerLog; import saker.build.task.Task; import saker.build.task.TaskContext; +import saker.build.task.TaskDependencyFuture; +import saker.build.task.TaskExecutionUtilities; import saker.build.task.TaskFactory; +import saker.build.task.dependencies.FileCollectionStrategy; import saker.build.task.identifier.TaskIdentifier; +import saker.build.task.utils.dependencies.RecursiveIgnoreCaseExtensionFileCollectionStrategy; import saker.build.thirdparty.saker.util.ImmutableUtils; import saker.build.thirdparty.saker.util.ObjectUtils; +import saker.build.thirdparty.saker.util.StringUtils; +import saker.build.thirdparty.saker.util.io.ByteArrayRegion; +import saker.build.thirdparty.saker.util.io.ByteSink; +import saker.build.thirdparty.saker.util.io.FileUtils; import saker.build.thirdparty.saker.util.io.SerialUtils; +import saker.build.thirdparty.saker.util.io.UnsyncByteArrayInputStream; +import saker.build.thirdparty.saker.util.io.UnsyncByteArrayOutputStream; +import saker.build.thirdparty.saker.util.io.function.IOConsumer; +import saker.java.compiler.api.classpath.ClassPathEntry; +import saker.java.compiler.api.classpath.ClassPathReference; +import saker.java.compiler.api.classpath.ClassPathVisitor; +import saker.java.compiler.api.classpath.CompilationClassPath; +import saker.java.compiler.api.classpath.FileClassPath; import saker.java.compiler.api.classpath.JavaClassPath; +import saker.java.compiler.api.classpath.JavaClassPathBuilder; +import saker.java.compiler.api.classpath.SDKClassPath; +import saker.java.compiler.api.compile.JavaCompilationWorkerTaskIdentifier; +import saker.java.compiler.api.compile.JavaCompilerWorkerTaskOutput; +import saker.java.compiler.api.compile.SakerJavaCompilerUtils; import saker.java.compiler.api.processor.ProcessorCreator; import saker.java.compiler.impl.JavaTaskUtils; import saker.java.compiler.impl.compile.CompileFileTags; import saker.java.compiler.impl.compile.WorkerJavaCompilerTaskFactoryBase; +import saker.java.compiler.impl.compile.util.LocalPathFileContentDescriptorExecutionProperty; import saker.java.compiler.impl.util.ClassPathEntryFileLocationExecutionProperty; +import saker.nest.bundle.NestBundleClassLoader; import saker.sdk.support.api.SDKDescription; +import saker.sdk.support.api.SDKPathReference; import saker.sdk.support.api.SDKReference; import saker.sdk.support.api.SDKSupportUtils; +import saker.std.api.file.location.ExecutionFileLocation; import saker.std.api.file.location.FileLocation; +import saker.std.api.file.location.FileLocationVisitor; +import saker.std.api.file.location.LocalFileLocation; public final class ProcessorCreatorTaskFactory implements TaskFactory, Task, Externalizable, TaskIdentifier { private static final long serialVersionUID = 1L; + //TODO we should clean up cached JARs that are old and no longer used. The MemoryTrimmer could be used to do that + + private static final FileTime FILE_TIME_ZERO = FileTime.fromMillis(0); + private String processorClassName; private JavaClassPath classPath; private NavigableMap sdks; @@ -72,13 +141,203 @@ public ProcessorCreatorTaskFactory(String processorClassName, JavaClassPath clas @Override public ProcessorCreator run(TaskContext taskcontext) throws Exception { Map sdkreferences = WorkerJavaCompilerTaskFactoryBase.toSDKReferences(taskcontext, sdks); - Set filelocations = collectFileLocations(taskcontext, classPath, sdkreferences); - return new ClassLoaderProcessorCreator(processorClassName, filelocations); + Map> localdirectorycontents = new HashMap<>(); + Map filelocations = collectFileLocations(taskcontext, classPath, sdkreferences, + localdirectorycontents::put); + Set cachedfiles = new LinkedHashSet<>(); + if (!filelocations.isEmpty()) { + NestBundleClassLoader cl = (NestBundleClassLoader) this.getClass().getClassLoader(); + Path jarcachedir = cl.getBundle().getBundleStoragePath().resolve("jar_cache"); + LocalFileProvider fp = LocalFileProvider.getInstance(); + fp.createDirectories(jarcachedir); + + for (Entry entry : filelocations.entrySet()) { + entry.getKey().accept(new FileLocationVisitor() { + @Override + public void visit(ExecutionFileLocation loc) { + SakerFile f = taskcontext.getTaskUtilities().resolveAtPath(loc.getPath()); + if (f == null) { + throw ObjectUtils + .sneakyThrow(new FileNotFoundException("Class path not found: " + loc.getPath())); + } + ByteArrayRegion filebytes; + if (f instanceof SakerDirectory) { + //a directory classpath. + //put all files in the classpath into a ZIP (JAR, but no manifest) + // and cache that + NavigableMap cpfiles = SakerPathFiles + .getFilesRecursiveByPath((SakerDirectory) f, DirectoryVisitPredicate.subFiles()); + try (UnsyncByteArrayOutputStream baos = new UnsyncByteArrayOutputStream()) { + try (ZipOutputStream zos = new ZipOutputStream(baos)) { + //don't compress, not necessary. performance is better + zos.setMethod(ZipOutputStream.DEFLATED); + zos.setLevel(0); + for (Entry fentry : cpfiles.entrySet()) { + ZipEntry zentry = new ZipEntry(fentry.getKey().toString()); + zentry.setCreationTime(FILE_TIME_ZERO); + zentry.setLastAccessTime(FILE_TIME_ZERO); + zentry.setLastModifiedTime(FILE_TIME_ZERO); + + zos.putNextEntry(zentry); + fentry.getValue().writeTo(zos); + zos.closeEntry(); + } + } catch (IOException e) { + throw ObjectUtils.sneakyThrow(e); + } + filebytes = baos.toByteArrayRegion(); + } + } else { + try { + filebytes = f.getBytes(); + } catch (IOException e) { + throw ObjectUtils.sneakyThrow(e); + } + } + try { + MessageDigest hash = FileUtils.getDefaultFileHasher(); + hash.update(filebytes.getArray(), filebytes.getOffset(), filebytes.getLength()); + Path cachejarpath = jarcachedir.resolve(StringUtils.toHexString(hash.digest()) + ".jar"); + + performJarCaching(fp, cachejarpath, cachedfiles, temp -> { + try (UnsyncByteArrayInputStream in = new UnsyncByteArrayInputStream(filebytes)) { + fp.writeToFile(in, temp); + } + }); + } catch (IOException e) { + throw ObjectUtils.sneakyThrow(e); + } + } + + @Override + public void visit(LocalFileLocation loc) { + SakerPath cppath = loc.getLocalPath(); + FileEntry fattrs; + try { + fattrs = fp.getFileAttributes(cppath); + } catch (IOException e) { + throw ObjectUtils.sneakyThrow(e); + } + if (fattrs.isDirectory()) { + NavigableSet dircontents = localdirectorycontents.get(loc); + if (dircontents == null) { + throw new ConcurrentModificationException( + "Class path was concurrently modified: " + loc); + } + ByteArrayRegion filebytes; + try (UnsyncByteArrayOutputStream baos = new UnsyncByteArrayOutputStream()) { + try (ZipOutputStream zos = new ZipOutputStream(baos)) { + ByteSink zipbytesink = ByteSink.valueOf(zos); + + //don't compress, not necessary. performance is better + zos.setMethod(ZipOutputStream.DEFLATED); + zos.setLevel(0); + for (SakerPath childpath : dircontents) { + SakerPath childabsolutepath = cppath.resolve(childpath); + + ZipEntry zentry = new ZipEntry(childpath.toString()); + zentry.setCreationTime(FILE_TIME_ZERO); + zentry.setLastAccessTime(FILE_TIME_ZERO); + zentry.setLastModifiedTime(FILE_TIME_ZERO); + + zos.putNextEntry(zentry); + fp.writeTo(childabsolutepath, zipbytesink); + zos.closeEntry(); + } + } catch (IOException e) { + throw ObjectUtils.sneakyThrow(e); + } + filebytes = baos.toByteArrayRegion(); + } + try { + MessageDigest hash = FileUtils.getDefaultFileHasher(); + hash.update(filebytes.getArray(), filebytes.getOffset(), filebytes.getLength()); + Path cachejarpath = jarcachedir + .resolve(StringUtils.toHexString(hash.digest()) + ".jar"); + + performJarCaching(fp, cachejarpath, cachedfiles, temp -> { + try (UnsyncByteArrayInputStream in = new UnsyncByteArrayInputStream(filebytes)) { + fp.writeToFile(in, temp); + } + }); + } catch (IOException e) { + throw ObjectUtils.sneakyThrow(e); + } + } else { + try { + FileHashResult hash = fp.hash(cppath, FileUtils.DEFAULT_FILE_HASH_ALGORITHM); + Path cachejarpath = jarcachedir + .resolve(StringUtils.toHexString(hash.getHash()) + ".jar"); + + performJarCaching(fp, cachejarpath, cachedfiles, temp -> { + try (InputStream in = fp.openInputStream(LocalFileProvider.toRealPath(cppath))) { + fp.writeToFile(in, temp); + } + }); + } catch (NoSuchAlgorithmException | IOException e) { + throw ObjectUtils.sneakyThrow(e); + } + } + } + + //suppress unused lock as it is only used for waiting + @SuppressWarnings("try") + private void performJarCaching(LocalFileProvider fp, Path cachejarpath, + Set cachedfiles, IOConsumer contentwriter) + throws IOException { + Path temp = cachejarpath.resolveSibling(cachejarpath.getFileName() + "_load"); + //synchronize to avoid overlapped locking + synchronized (("saker.java.compiler.jar_cache:" + cachejarpath).intern()) { + try { + fp.getFileAttributes(cachejarpath); + //the file exists. we don't check if it is a regular file or not, as if it is not, + //then somebody did something wrong with the storage. that should be fixed by the user + cachedfiles.add(LocalFileLocation.create(SakerPath.valueOf(cachejarpath))); + } catch (IOException e) { + Path lockpath = cachejarpath.resolveSibling(cachejarpath.getFileName() + "_lock"); + try (FileChannel fc = FileChannel.open(lockpath, StandardOpenOption.CREATE, + StandardOpenOption.WRITE, StandardOpenOption.READ)) { + try (FileLock lock = fc.tryLock()) { + if (lock == null) { + //it is already locked by some other agent on the machine. might be when concurrently loading + //in this case wait the other agent to load by locking again + try (FileLock l2 = fc.lock()) { + //do nothing. the lock waits for it to complete + //the cached jar should be in a valid state when we receive the lock + //can return without doing anything + cachedfiles + .add(LocalFileLocation.create(SakerPath.valueOf(cachejarpath))); + return; + } + } + //we got the lock, and should write to the temp file + contentwriter.accept(temp); + try { + Files.move(temp, cachejarpath); + cachedfiles.add(LocalFileLocation.create(SakerPath.valueOf(cachejarpath))); + } catch (FileAlreadyExistsException fae) { + // somebody put there something that we didn't notice + //its fine, we use that + cachedfiles.add(LocalFileLocation.create(SakerPath.valueOf(cachejarpath))); + return; + } + } + } finally { + Files.deleteIfExists(temp); + } + } + } + } + }); + } + } + return new ClassLoaderProcessorCreator(processorClassName, cachedfiles); } - private static Set collectFileLocations(TaskContext taskcontext, JavaClassPath classpath, - Map sdkreferences) throws IOException { - return JavaTaskUtils.collectFileLocationsWithImplementationDependencyReporting(taskcontext, classpath, + private static Map collectFileLocations(TaskContext taskcontext, + JavaClassPath classpath, Map sdkreferences, + BiConsumer> localdircontentsconsumer) throws IOException { + return collectFileLocationsWithImplementationDependencyReporting(taskcontext, classpath, CompileFileTags.INPUT_CLASSPATH, sdkreferences, entry -> { //we need to add a dependency on the file locations returned from the ClassPathEntry instances //if we don't, then the entries can return different file locations while the classpath is still the same @@ -86,7 +345,7 @@ private static Set collectFileLocations(TaskContext taskcontext, J //e.g. if a nest bundle is moved from pending to local, the classpath doesn't change only the file location return taskcontext.getTaskUtilities() .getReportExecutionDependency(new ClassPathEntryFileLocationExecutionProperty(entry)); - }).keySet(); + }, localdircontentsconsumer); } @Override @@ -94,6 +353,165 @@ public Task createTask(ExecutionContext executioncon return this; } + /** + * @return The content descriptors may be null. + */ + public static Map collectFileLocationsWithImplementationDependencyReporting( + TaskContext taskcontext, JavaClassPath classpath, Object tag, Map sdks, + Function classpathentryfilelocationhandler, + BiConsumer> localdircontentsconsumer) throws IOException { + if (classpath == null) { + return Collections.emptyMap(); + } + Map result = new LinkedHashMap<>(); + classpath.accept(new ClassPathVisitor() { + private Set handledWorkerTaskIds = new HashSet<>(); + + @Override + public void visit(ClassPathReference classpath) { + Collection entries = classpath.getEntries(); + if (ObjectUtils.isNullOrEmpty(entries)) { + SakerLog.warning().println("No class path entries found for: " + classpath); + return; + } + for (ClassPathEntry entry : entries) { + if (entry == null) { + SakerLog.warning().println("Class path entry is null for: " + classpath); + continue; + } + FileLocation filelocation = classpathentryfilelocationhandler.apply(entry); + if (filelocation == null) { + SakerLog.warning().println("No class path file location for: " + entry); + continue; + } + handleFileLocation(filelocation); + + Collection additionalclasspaths = entry + .getAdditionalClassPathReferences(); + if (!ObjectUtils.isNullOrEmpty(additionalclasspaths)) { + JavaClassPathBuilder additionalcpbuilder = JavaClassPathBuilder.newBuilder(); + for (ClassPathReference additionalcp : additionalclasspaths) { + additionalcpbuilder.addClassPath(additionalcp); + } + JavaClassPath additionalcp = additionalcpbuilder.build(); + additionalcp.accept(this); + } + } + } + + @Override + public void visit(CompilationClassPath classpath) { + JavaCompilationWorkerTaskIdentifier workertaskid = classpath.getCompilationWorkerTaskIdentifier(); + if (!handledWorkerTaskIds.add(workertaskid)) { + //don't get the task result to not install another dependency + return; + } + TaskDependencyFuture depresult = taskcontext.getTaskDependencyFuture(workertaskid); + JavaCompilerWorkerTaskOutput output = (JavaCompilerWorkerTaskOutput) depresult.getFinished(); + SakerPath classdirpath = output.getClassDirectory(); + ExecutionFileLocation filelocation = ExecutionFileLocation.create(classdirpath); + JavaClassPath outputcp = output.getClassPath(); + + Object implversionkey = output.getImplementationVersionKey(); + if (implversionkey != null) { + depresult.setTaskOutputChangeDetector(SakerJavaCompilerUtils + .getCompilerOutputImplementationVersionKeyTaskOutputChangeDetector(implversionkey)); + depresult.setTaskOutputChangeDetector( + SakerJavaCompilerUtils.getCompilerOutputClassPathTaskOutputChangeDetector(outputcp)); + result.put(filelocation, new SerializableContentDescriptor(implversionkey)); + } else { + SakerDirectory classesdir = taskcontext.getTaskUtilities().resolveDirectoryAtPath(classdirpath); + if (classesdir == null) { + throw ObjectUtils.sneakyThrow( + new FileNotFoundException("Compilation class directory not found: " + classesdir)); + } + + FileCollectionStrategy classfileadditiondep = RecursiveIgnoreCaseExtensionFileCollectionStrategy + .create(classdirpath, "." + JavaTaskUtils.EXTENSION_CLASSFILE); + NavigableMap classfiles = taskcontext.getTaskUtilities() + .collectFilesReportInputFileAndAdditionDependency(tag, classfileadditiondep); + + NavigableMap contentmap = SakerPathFiles.toFileContentMap(classfiles); + result.put(filelocation, new MultiPathContentDescriptor(contentmap)); + } + if (outputcp != null) { + outputcp.accept(this); + } + } + + @Override + public void visit(FileClassPath classpath) { + FileLocation location = classpath.getFileLocation(); + handleFileLocation(location); + } + + @Override + public void visit(SDKClassPath classpath) { + SDKPathReference sdkpathref = classpath.getSDKPathReference(); + SakerPath path = SDKSupportUtils.getSDKPathReferencePath(sdkpathref, sdks); + LocalFileLocation fileloc = LocalFileLocation.create(path); + result.put(fileloc, null); + } + + private ContentDescriptor handleExecutionFileLocation(SakerPath path, SakerFile cpfile) { + if (cpfile instanceof SakerDirectory) { + FileCollectionStrategy classfileadditiondep = RecursiveIgnoreCaseExtensionFileCollectionStrategy + .create(path, "." + JavaTaskUtils.EXTENSION_CLASSFILE); + NavigableMap classfiles = taskcontext.getTaskUtilities() + .collectFilesReportInputFileAndAdditionDependency(tag, classfileadditiondep); + return new MultiPathContentDescriptor(SakerPathFiles.toFileContentMap(classfiles)); + } + taskcontext.getTaskUtilities().reportInputFileDependency(tag, cpfile); + return cpfile.getContentDescriptor(); + } + + private void handleFileLocation(FileLocation location) { + if (result.containsKey(location)) { + return; + } + ContentDescriptor[] cdres = { null }; + location.accept(new FileLocationVisitor() { + @Override + public void visit(ExecutionFileLocation loc) { + SakerPath path = loc.getPath(); + SakerFile cpfile = taskcontext.getTaskUtilities().resolveAtPath(path); + if (cpfile == null) { + throw ObjectUtils + .sneakyThrow(new FileNotFoundException("Class path file not found: " + path)); + } + cdres[0] = handleExecutionFileLocation(path, cpfile); + } + + @Override + public void visit(LocalFileLocation loc) { + SakerPath path = loc.getLocalPath(); + TaskExecutionUtilities taskutils = taskcontext.getTaskUtilities(); + ContentDescriptor cd = taskutils.getReportExecutionDependency( + new LocalPathFileContentDescriptorExecutionProperty(path)); + if (cd == null) { + throw ObjectUtils + .sneakyThrow(new FileNotFoundException("Class path local file not found: " + path)); + } + + if (DirectoryContentDescriptor.INSTANCE.equals(cd)) { + //the class path denotes a directory + //add the dependencies on the files (all files, not only .class files) + + LocalDirectoryFilesExecutionProperty.PropertyValue pval = taskutils + .getReportExecutionDependency(new LocalDirectoryFilesExecutionProperty(path)); + cdres[0] = new MultiPathContentDescriptor(pval.getContents()); + localdircontentsconsumer.accept(loc, pval.getContents().navigableKeySet()); + } else { + cdres[0] = cd; + } + } + }); + result.put(location, cdres[0]); + } + }); + return result; + } + @Override public void writeExternal(ObjectOutput out) throws IOException { out.writeObject(processorClassName); @@ -151,4 +569,140 @@ public String toString() { + (classPath != null ? "classPath=" + classPath + ", " : "") + (sdks != null ? "sdks=" + sdks : "") + "]"; } + + public static class LocalDirectoryFilesExecutionProperty + implements ExecutionProperty, Externalizable { + private static final long serialVersionUID = 1L; + + public static class PropertyValue implements Externalizable { + private static final long serialVersionUID = 1L; + + private NavigableMap contents; + + /** + * For {@link Externalizable}. + */ + public PropertyValue() { + } + + public PropertyValue(NavigableMap contents) { + this.contents = contents; + } + + public NavigableMap getContents() { + return contents; + } + + @Override + public void writeExternal(ObjectOutput out) throws IOException { + SerialUtils.writeExternalMap(out, contents); + } + + @Override + public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + contents = SerialUtils.readExternalImmutableNavigableMap(in); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((contents == null) ? 0 : contents.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + PropertyValue other = (PropertyValue) obj; + if (!ObjectUtils.mapOrderedEquals(this.contents, other.contents)) { + return false; + } + return true; + } + + @Override + public String toString() { + return getClass().getSimpleName() + "[" + (contents != null ? "contents=" + contents : "") + "]"; + } + + } + + private SakerPath path; + + /** + * For {@link Externalizable}. + */ + public LocalDirectoryFilesExecutionProperty() { + } + + public LocalDirectoryFilesExecutionProperty(SakerPath path) { + this.path = path; + } + + @Override + public PropertyValue getCurrentValue(ExecutionContext executioncontext) throws Exception { + NavigableMap result = new TreeMap<>(); + for (Entry entry : LocalFileProvider.getInstance() + .getDirectoryEntriesRecursively(path).entrySet()) { + if (!entry.getValue().isRegularFile()) { + continue; + } + SakerPath keypath = entry.getKey(); + SakerPath cpabspath = path.resolve(keypath); + ContentDescriptor classfilecd = executioncontext.getExecutionPropertyCurrentValue( + new LocalPathFileContentDescriptorExecutionProperty(cpabspath)); + if (classfilecd == null) { + continue; + } + result.put(cpabspath, classfilecd); + } + return new PropertyValue(ImmutableUtils.unmodifiableNavigableMap(result)); + } + + @Override + public void writeExternal(ObjectOutput out) throws IOException { + out.writeObject(path); + } + + @Override + public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + path = (SakerPath) in.readObject(); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((path == null) ? 0 : path.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + LocalDirectoryFilesExecutionProperty other = (LocalDirectoryFilesExecutionProperty) obj; + if (path == null) { + if (other.path != null) + return false; + } else if (!path.equals(other.path)) + return false; + return true; + } + + @Override + public String toString() { + return getClass().getSimpleName() + "[" + path + "]"; + } + } } \ No newline at end of file From e9cf8ddf6946d784594ddb4d7c8ab583c210c1c6 Mon Sep 17 00:00:00 2001 From: sipkab <10866741+Sipkab@users.noreply.github.com> Date: Thu, 9 Jan 2020 19:19:51 +0100 Subject: [PATCH 3/5] Add API for representing immutable classpath in ClassPathEntry --- .../api/classpath/ClassPathEntry.java | 22 ++++++ .../processor/ProcessorCreationContext.java | 2 +- .../handler/ProcessorCreationContextImpl.java | 29 ------- .../ClassLoaderProcessorCreator.java | 75 +++++++++++++------ .../ProcessorCreatorTaskFactory.java | 30 +++++++- 5 files changed, 103 insertions(+), 55 deletions(-) diff --git a/api/src/main/saker/java/compiler/api/classpath/ClassPathEntry.java b/api/src/main/saker/java/compiler/api/classpath/ClassPathEntry.java index 47f535d..5488408 100644 --- a/api/src/main/saker/java/compiler/api/classpath/ClassPathEntry.java +++ b/api/src/main/saker/java/compiler/api/classpath/ClassPathEntry.java @@ -128,6 +128,28 @@ public default StructuredTaskResult getDocumentationAttachment() { */ public Object getImplementationVersionKey(); + /** + * Checks if this classpath should be considered immutable. + *

+ * An immutable classpaths is one that is expected to never change in any way. This includes that the + * classpath contents doesn't change during the build and between builds. The classpath contents are expected to be + * the same after the initial construction of the classpath. + *

+ * The consumer of this classpath may lock the contents of the classpath in ways that prevent its mutability. If a + * classpath is reported as immutable, build errors should be expected if it is modified. + *

+ * An example for these are classpaths from SDKs, release artifacts from repositories, and others. These are + * expected to not be modified after they've been published. + *

+ * Using immutable classpaths can improve performance as various tasks may not need to cache them in an off-site + * location, but can use them in-place as that doesn't distrupt the workflow. + * + * @return true if the classpath is immutable. + */ + public default boolean isImmutable() { + return false; + } + @Override public int hashCode(); diff --git a/api/src/main/saker/java/compiler/api/processor/ProcessorCreationContext.java b/api/src/main/saker/java/compiler/api/processor/ProcessorCreationContext.java index fc6250b..6dbeb92 100644 --- a/api/src/main/saker/java/compiler/api/processor/ProcessorCreationContext.java +++ b/api/src/main/saker/java/compiler/api/processor/ProcessorCreationContext.java @@ -39,4 +39,4 @@ public interface ProcessorCreationContext { * @return The environment. */ public SakerEnvironment getEnvironment(); -} + } diff --git a/impl/src/main/saker/java/compiler/impl/compile/handler/ProcessorCreationContextImpl.java b/impl/src/main/saker/java/compiler/impl/compile/handler/ProcessorCreationContextImpl.java index 3bdb41d..d5315d3 100644 --- a/impl/src/main/saker/java/compiler/impl/compile/handler/ProcessorCreationContextImpl.java +++ b/impl/src/main/saker/java/compiler/impl/compile/handler/ProcessorCreationContextImpl.java @@ -15,14 +15,7 @@ */ package saker.java.compiler.impl.compile.handler; -import java.io.IOException; -import java.nio.file.Path; - -import saker.build.exception.FileMirroringUnavailableException; -import saker.build.file.DirectoryVisitPredicate; -import saker.build.file.SakerFile; import saker.build.runtime.environment.SakerEnvironment; -import saker.build.runtime.execution.ExecutionContext; import saker.build.task.TaskContext; import saker.java.compiler.api.processor.ProcessorCreationContext; @@ -33,28 +26,6 @@ public ProcessorCreationContextImpl(TaskContext taskContext) { this.taskContext = taskContext; } - @Deprecated - public ExecutionContext getExecutionContext() { - return taskContext.getExecutionContext(); - } - - /** - * @see TaskContext#mirror(SakerFile, DirectoryVisitPredicate) - */ - @Deprecated - public Path mirror(SakerFile file, DirectoryVisitPredicate synchpredicate) - throws IOException, NullPointerException, FileMirroringUnavailableException { - return taskContext.mirror(file, synchpredicate); - } - - /** - * @see TaskContext#mirror(SakerFile) - */ - @Deprecated - public Path mirror(SakerFile file) throws IOException, NullPointerException, FileMirroringUnavailableException { - return mirror(file, DirectoryVisitPredicate.everything()); - } - @Override public SakerEnvironment getEnvironment() { return taskContext.getExecutionContext().getEnvironment(); diff --git a/impl/src/main/saker/java/compiler/impl/processor/ClassLoaderProcessorCreator.java b/impl/src/main/saker/java/compiler/impl/processor/ClassLoaderProcessorCreator.java index 0d0cd18..356cdb1 100644 --- a/impl/src/main/saker/java/compiler/impl/processor/ClassLoaderProcessorCreator.java +++ b/impl/src/main/saker/java/compiler/impl/processor/ClassLoaderProcessorCreator.java @@ -15,11 +15,14 @@ */ package saker.java.compiler.impl.processor; +import java.io.Closeable; import java.io.Externalizable; import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; import java.lang.reflect.Constructor; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.LinkedHashSet; import java.util.Objects; @@ -35,6 +38,7 @@ import saker.build.thirdparty.saker.util.classloader.JarClassLoaderDataFinder; import saker.build.thirdparty.saker.util.classloader.MultiClassLoader; import saker.build.thirdparty.saker.util.classloader.MultiDataClassLoader; +import saker.build.thirdparty.saker.util.classloader.PathClassLoaderDataFinder; import saker.build.thirdparty.saker.util.io.JarFileUtils; import saker.build.thirdparty.saker.util.io.SerialUtils; import saker.build.util.cache.CacheKey; @@ -47,7 +51,7 @@ public final class ClassLoaderProcessorCreator implements ProcessorCreator, Exte private static final long serialVersionUID = 1L; private String className; - private Set classPathJarFileLocations; + private Set classPathFileLocations; /** * For {@link Externalizable}. @@ -55,12 +59,12 @@ public final class ClassLoaderProcessorCreator implements ProcessorCreator, Exte public ClassLoaderProcessorCreator() { } - public ClassLoaderProcessorCreator(String className, Set classPathJarFileLocations) { + public ClassLoaderProcessorCreator(String className, Set classPathFileLocations) { Objects.requireNonNull(className, "className"); - Objects.requireNonNull(classPathJarFileLocations, "classPathJarFileLocations"); + Objects.requireNonNull(classPathFileLocations, "classPathFileLocations"); this.className = className; - this.classPathJarFileLocations = classPathJarFileLocations; + this.classPathFileLocations = classPathFileLocations; } @Override @@ -72,21 +76,20 @@ public String getName() { public Processor create(ProcessorCreationContext creationcontext) throws Exception { SakerEnvironment environment = creationcontext.getEnvironment(); Constructor constructor = environment - .getCachedData(new ProcessorConstructorCacheKey(environment, className, - classPathJarFileLocations)); + .getCachedData(new ProcessorConstructorCacheKey(environment, className, classPathFileLocations)); return constructor.newInstance(); } @Override public void writeExternal(ObjectOutput out) throws IOException { out.writeUTF(className); - SerialUtils.writeExternalCollection(out, classPathJarFileLocations); + SerialUtils.writeExternalCollection(out, classPathFileLocations); } @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { className = in.readUTF(); - classPathJarFileLocations = SerialUtils.readExternalImmutableLinkedHashSet(in); + classPathFileLocations = SerialUtils.readExternalImmutableLinkedHashSet(in); } @Override @@ -94,7 +97,7 @@ public int hashCode() { final int prime = 31; int result = 1; result = prime * result + ((className == null) ? 0 : className.hashCode()); - result = prime * result + ((classPathJarFileLocations == null) ? 0 : classPathJarFileLocations.hashCode()); + result = prime * result + ((classPathFileLocations == null) ? 0 : classPathFileLocations.hashCode()); return result; } @@ -112,10 +115,10 @@ public boolean equals(Object obj) { return false; } else if (!className.equals(other.className)) return false; - if (classPathJarFileLocations == null) { - if (other.classPathJarFileLocations != null) + if (classPathFileLocations == null) { + if (other.classPathFileLocations != null) return false; - } else if (!classPathJarFileLocations.equals(other.classPathJarFileLocations)) + } else if (!classPathFileLocations.equals(other.classPathFileLocations)) return false; return true; } @@ -123,10 +126,15 @@ public boolean equals(Object obj) { @Override public String toString() { return getClass().getSimpleName() + "[className=" + className + ", classPathJarFileLocations=" - + classPathJarFileLocations + "]"; + + classPathFileLocations + "]"; } - private static class JarClassLoaderCacheKey implements CacheKey { + private interface ClassLoaderDataFinderCreator extends Closeable { + public ClassLoaderDataFinder create(); + } + + private static class JarClassLoaderCacheKey + implements CacheKey { private LocalFileLocation fileLocation; public JarClassLoaderCacheKey(LocalFileLocation fileLocation) { @@ -134,18 +142,43 @@ public JarClassLoaderCacheKey(LocalFileLocation fileLocation) { } @Override - public JarFile allocate() throws Exception { - return JarFileUtils.createMultiReleaseJarFile(LocalFileProvider.toRealPath(fileLocation.getLocalPath())); + public ClassLoaderDataFinderCreator allocate() throws Exception { + Path realpath = LocalFileProvider.toRealPath(fileLocation.getLocalPath()); + if (Files.isDirectory(realpath)) { + return new ClassLoaderDataFinderCreator() { + @Override + public void close() throws IOException { + } + + @Override + public ClassLoaderDataFinder create() { + return new PathClassLoaderDataFinder(realpath); + } + }; + } + return new ClassLoaderDataFinderCreator() { + private final JarFile jarfile = JarFileUtils.createMultiReleaseJarFile(realpath); + + @Override + public void close() throws IOException { + jarfile.close(); + } + + @Override + public ClassLoaderDataFinder create() { + return new JarClassLoaderDataFinder(jarfile); + } + }; } @Override - public void close(JarClassLoaderDataFinder data, JarFile resource) throws Exception { + public void close(ClassLoaderDataFinder data, ClassLoaderDataFinderCreator resource) throws Exception { resource.close(); } @Override - public JarClassLoaderDataFinder generate(JarFile resource) throws Exception { - return new JarClassLoaderDataFinder(resource); + public ClassLoaderDataFinder generate(ClassLoaderDataFinderCreator resource) throws Exception { + return resource.create(); } @Override @@ -155,7 +188,7 @@ public long getExpiry() { } @Override - public boolean validate(JarClassLoaderDataFinder data, JarFile resource) { + public boolean validate(ClassLoaderDataFinder data, ClassLoaderDataFinderCreator resource) { return true; } } @@ -191,7 +224,7 @@ public void close(Constructor data, Object resource) throws public Constructor generate(Object resource) throws Exception { Set datafinders = new LinkedHashSet<>(); for (JarClassLoaderCacheKey ck : cacheKeys) { - JarClassLoaderDataFinder jarcldf = environment.getCachedData(ck); + ClassLoaderDataFinder jarcldf = environment.getCachedData(ck); datafinders.add(jarcldf); } MultiDataClassLoader cl = new MultiDataClassLoader(PROCESSOR_PARENT_CLASSLOADER, datafinders); diff --git a/impl/src/main/saker/java/compiler/impl/processor/ProcessorCreatorTaskFactory.java b/impl/src/main/saker/java/compiler/impl/processor/ProcessorCreatorTaskFactory.java index 9264886..17ee96b 100644 --- a/impl/src/main/saker/java/compiler/impl/processor/ProcessorCreatorTaskFactory.java +++ b/impl/src/main/saker/java/compiler/impl/processor/ProcessorCreatorTaskFactory.java @@ -44,6 +44,7 @@ import java.util.Set; import java.util.TreeMap; import java.util.function.BiConsumer; +import java.util.function.Consumer; import java.util.function.Function; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -142,8 +143,9 @@ public ProcessorCreatorTaskFactory(String processorClassName, JavaClassPath clas public ProcessorCreator run(TaskContext taskcontext) throws Exception { Map sdkreferences = WorkerJavaCompilerTaskFactoryBase.toSDKReferences(taskcontext, sdks); Map> localdirectorycontents = new HashMap<>(); + Set immutableclasspaths = new HashSet<>(); Map filelocations = collectFileLocations(taskcontext, classPath, sdkreferences, - localdirectorycontents::put); + localdirectorycontents::put, immutableclasspaths::add); Set cachedfiles = new LinkedHashSet<>(); if (!filelocations.isEmpty()) { NestBundleClassLoader cl = (NestBundleClassLoader) this.getClass().getClassLoader(); @@ -155,6 +157,16 @@ public ProcessorCreator run(TaskContext taskcontext) throws Exception { entry.getKey().accept(new FileLocationVisitor() { @Override public void visit(ExecutionFileLocation loc) { + if (immutableclasspaths.contains(loc)) { + Path localpath = taskcontext.getExecutionContext().getPathConfiguration() + .toLocalPath(loc.getPath()); + if (localpath != null) { + cachedfiles.add(LocalFileLocation.create(SakerPath.valueOf(localpath))); + return; + } + //its not a local path, we can't use it. we need to copy it to the local storage + } + SakerFile f = taskcontext.getTaskUtilities().resolveAtPath(loc.getPath()); if (f == null) { throw ObjectUtils @@ -211,6 +223,10 @@ public void visit(ExecutionFileLocation loc) { @Override public void visit(LocalFileLocation loc) { + if (immutableclasspaths.contains(loc)) { + cachedfiles.add(loc); + return; + } SakerPath cppath = loc.getLocalPath(); FileEntry fattrs; try { @@ -336,7 +352,8 @@ private void performJarCaching(LocalFileProvider fp, Path cachejarpath, private static Map collectFileLocations(TaskContext taskcontext, JavaClassPath classpath, Map sdkreferences, - BiConsumer> localdircontentsconsumer) throws IOException { + BiConsumer> localdircontentsconsumer, + Consumer immutableclasspathconsumer) throws IOException { return collectFileLocationsWithImplementationDependencyReporting(taskcontext, classpath, CompileFileTags.INPUT_CLASSPATH, sdkreferences, entry -> { //we need to add a dependency on the file locations returned from the ClassPathEntry instances @@ -345,7 +362,7 @@ private static Map collectFileLocations(TaskCon //e.g. if a nest bundle is moved from pending to local, the classpath doesn't change only the file location return taskcontext.getTaskUtilities() .getReportExecutionDependency(new ClassPathEntryFileLocationExecutionProperty(entry)); - }, localdircontentsconsumer); + }, localdircontentsconsumer, immutableclasspathconsumer); } @Override @@ -359,7 +376,8 @@ public Task createTask(ExecutionContext executioncon public static Map collectFileLocationsWithImplementationDependencyReporting( TaskContext taskcontext, JavaClassPath classpath, Object tag, Map sdks, Function classpathentryfilelocationhandler, - BiConsumer> localdircontentsconsumer) throws IOException { + BiConsumer> localdircontentsconsumer, + Consumer immutableclasspathconsumer) throws IOException { if (classpath == null) { return Collections.emptyMap(); } @@ -384,6 +402,9 @@ public void visit(ClassPathReference classpath) { SakerLog.warning().println("No class path file location for: " + entry); continue; } + if (entry.isImmutable()) { + immutableclasspathconsumer.accept(filelocation); + } handleFileLocation(filelocation); Collection additionalclasspaths = entry @@ -451,6 +472,7 @@ public void visit(SDKClassPath classpath) { SakerPath path = SDKSupportUtils.getSDKPathReferencePath(sdkpathref, sdks); LocalFileLocation fileloc = LocalFileLocation.create(path); result.put(fileloc, null); + immutableclasspathconsumer.accept(fileloc); } private ContentDescriptor handleExecutionFileLocation(SakerPath path, SakerFile cpfile) { From 0dd1e076440e10aaa2c6bed598f7423e8c7848e9 Mon Sep 17 00:00:00 2001 From: sipkab <10866741+Sipkab@users.noreply.github.com> Date: Thu, 9 Jan 2020 20:33:02 +0100 Subject: [PATCH 4/5] Rename immutable class path property to static --- .../api/classpath/ClassPathEntry.java | 29 ++++++++++++------- .../bundle/BundleClassPathEntry.java | 6 ++++ .../ProcessorCreatorTaskFactory.java | 20 ++++++------- 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/api/src/main/saker/java/compiler/api/classpath/ClassPathEntry.java b/api/src/main/saker/java/compiler/api/classpath/ClassPathEntry.java index 5488408..23aaf7d 100644 --- a/api/src/main/saker/java/compiler/api/classpath/ClassPathEntry.java +++ b/api/src/main/saker/java/compiler/api/classpath/ClassPathEntry.java @@ -129,24 +129,33 @@ public default StructuredTaskResult getDocumentationAttachment() { public Object getImplementationVersionKey(); /** - * Checks if this classpath should be considered immutable. + * Checks if the {@linkplain #getFileLocation() file location} of this classpath may be considered static. *

- * An immutable classpaths is one that is expected to never change in any way. This includes that the - * classpath contents doesn't change during the build and between builds. The classpath contents are expected to be - * the same after the initial construction of the classpath. + * A static classpath is one that doesn't change during the lifetime of the enclosing build environment. The files + * of a static classpath will not be attempted to be modified by other agents on the same computer. If the classpath + * represents a directory, then the enclosed files in the directory mustn't change. If it is a JAR, or other + * archive, then the file itself mustn't change. *

- * The consumer of this classpath may lock the contents of the classpath in ways that prevent its mutability. If a - * classpath is reported as immutable, build errors should be expected if it is modified. + * If a classpath is static, that means that the users of the classpath are allowed to load the files of the + * classpath directly from its location and doesn't need to copy it elsewhere. As the classpath is opened, + * modifications may be blocked to them by the operating system. *

- * An example for these are classpaths from SDKs, release artifacts from repositories, and others. These are + * E.g. if a static JAR path is opened by the build environment, and the user attempts to delete, rename, modify, or + * otherwise manipulate the JAR, then it may fail, as the build environment loaded it. + *

+ * In order to modify static classpaths, the user may need to reload the build environment. If that is distruptive + * to the normal workflow, then the classpath shouldn't be considered static. + *

+ * An example for static classpaths are classpaths from SDKs, artifacts from repositories, and others. These are * expected to not be modified after they've been published. *

- * Using immutable classpaths can improve performance as various tasks may not need to cache them in an off-site + * Using static classpaths can improve performance as various tasks may not need to cache them in an off-site * location, but can use them in-place as that doesn't distrupt the workflow. * - * @return true if the classpath is immutable. + * @return true if the classpath is static. + * @since saker.java.compiler 0.8.1 */ - public default boolean isImmutable() { + public default boolean isStaticFile() { return false; } diff --git a/impl/src/main/saker/java/compiler/impl/classpath/bundle/BundleClassPathEntry.java b/impl/src/main/saker/java/compiler/impl/classpath/bundle/BundleClassPathEntry.java index 96bda11..72576d1 100644 --- a/impl/src/main/saker/java/compiler/impl/classpath/bundle/BundleClassPathEntry.java +++ b/impl/src/main/saker/java/compiler/impl/classpath/bundle/BundleClassPathEntry.java @@ -95,6 +95,12 @@ public FileLocation getFileLocation() { return LocalFileLocation.create(SakerPath.valueOf(jarbundle.getJarPath())); } + @Override + public boolean isStaticFile() { + //the bundle files doesn't change during the lifetime of the build environment, so they are static + return true; + } + @Override public Collection getAdditionalClassPathReferences() { return null; diff --git a/impl/src/main/saker/java/compiler/impl/processor/ProcessorCreatorTaskFactory.java b/impl/src/main/saker/java/compiler/impl/processor/ProcessorCreatorTaskFactory.java index 17ee96b..5df8644 100644 --- a/impl/src/main/saker/java/compiler/impl/processor/ProcessorCreatorTaskFactory.java +++ b/impl/src/main/saker/java/compiler/impl/processor/ProcessorCreatorTaskFactory.java @@ -143,9 +143,9 @@ public ProcessorCreatorTaskFactory(String processorClassName, JavaClassPath clas public ProcessorCreator run(TaskContext taskcontext) throws Exception { Map sdkreferences = WorkerJavaCompilerTaskFactoryBase.toSDKReferences(taskcontext, sdks); Map> localdirectorycontents = new HashMap<>(); - Set immutableclasspaths = new HashSet<>(); + Set staticclasspaths = new HashSet<>(); Map filelocations = collectFileLocations(taskcontext, classPath, sdkreferences, - localdirectorycontents::put, immutableclasspaths::add); + localdirectorycontents::put, staticclasspaths::add); Set cachedfiles = new LinkedHashSet<>(); if (!filelocations.isEmpty()) { NestBundleClassLoader cl = (NestBundleClassLoader) this.getClass().getClassLoader(); @@ -157,7 +157,7 @@ public ProcessorCreator run(TaskContext taskcontext) throws Exception { entry.getKey().accept(new FileLocationVisitor() { @Override public void visit(ExecutionFileLocation loc) { - if (immutableclasspaths.contains(loc)) { + if (staticclasspaths.contains(loc)) { Path localpath = taskcontext.getExecutionContext().getPathConfiguration() .toLocalPath(loc.getPath()); if (localpath != null) { @@ -223,7 +223,7 @@ public void visit(ExecutionFileLocation loc) { @Override public void visit(LocalFileLocation loc) { - if (immutableclasspaths.contains(loc)) { + if (staticclasspaths.contains(loc)) { cachedfiles.add(loc); return; } @@ -353,7 +353,7 @@ private void performJarCaching(LocalFileProvider fp, Path cachejarpath, private static Map collectFileLocations(TaskContext taskcontext, JavaClassPath classpath, Map sdkreferences, BiConsumer> localdircontentsconsumer, - Consumer immutableclasspathconsumer) throws IOException { + Consumer staticclasspathconsumer) throws IOException { return collectFileLocationsWithImplementationDependencyReporting(taskcontext, classpath, CompileFileTags.INPUT_CLASSPATH, sdkreferences, entry -> { //we need to add a dependency on the file locations returned from the ClassPathEntry instances @@ -362,7 +362,7 @@ private static Map collectFileLocations(TaskCon //e.g. if a nest bundle is moved from pending to local, the classpath doesn't change only the file location return taskcontext.getTaskUtilities() .getReportExecutionDependency(new ClassPathEntryFileLocationExecutionProperty(entry)); - }, localdircontentsconsumer, immutableclasspathconsumer); + }, localdircontentsconsumer, staticclasspathconsumer); } @Override @@ -377,7 +377,7 @@ public static Map collectFileLocationsWithImple TaskContext taskcontext, JavaClassPath classpath, Object tag, Map sdks, Function classpathentryfilelocationhandler, BiConsumer> localdircontentsconsumer, - Consumer immutableclasspathconsumer) throws IOException { + Consumer staticlasspathconsumer) throws IOException { if (classpath == null) { return Collections.emptyMap(); } @@ -402,8 +402,8 @@ public void visit(ClassPathReference classpath) { SakerLog.warning().println("No class path file location for: " + entry); continue; } - if (entry.isImmutable()) { - immutableclasspathconsumer.accept(filelocation); + if (entry.isStaticFile()) { + staticlasspathconsumer.accept(filelocation); } handleFileLocation(filelocation); @@ -472,7 +472,7 @@ public void visit(SDKClassPath classpath) { SakerPath path = SDKSupportUtils.getSDKPathReferencePath(sdkpathref, sdks); LocalFileLocation fileloc = LocalFileLocation.create(path); result.put(fileloc, null); - immutableclasspathconsumer.accept(fileloc); + staticlasspathconsumer.accept(fileloc); } private ContentDescriptor handleExecutionFileLocation(SakerPath path, SakerFile cpfile) { From cdec6333cf738f5a7984f3a0f8ea2314daf0d31c Mon Sep 17 00:00:00 2001 From: sipkab <10866741+Sipkab@users.noreply.github.com> Date: Thu, 9 Jan 2020 21:26:00 +0100 Subject: [PATCH 5/5] Report execution property dependency on cached jar outputs So if they are deleted between build executions, they are re-cached again. --- .../java/compiler/impl/JavaTaskUtils.java | 16 ++++++++--- .../handler/full/FullCompilationHandler.java | 4 +-- .../IncrementalCompilationHandler.java | 10 ++++--- ...ileContentDescriptorExecutionProperty.java | 14 +++++++++- .../ProcessorCreatorTaskFactory.java | 28 ++++++++++++++++--- 5 files changed, 57 insertions(+), 15 deletions(-) diff --git a/impl/src/main/saker/java/compiler/impl/JavaTaskUtils.java b/impl/src/main/saker/java/compiler/impl/JavaTaskUtils.java index 4a9bef3..85d5c60 100644 --- a/impl/src/main/saker/java/compiler/impl/JavaTaskUtils.java +++ b/impl/src/main/saker/java/compiler/impl/JavaTaskUtils.java @@ -185,8 +185,6 @@ public static Collection toAddExportsCommandLineStrings(JavaAddExports a return SakerJavaCompilerUtils.toAddExportsCommandLineStrings(addexports); } - - public static class LocalDirectoryClassFilesExecutionProperty implements ExecutionProperty, Externalizable { private static final long serialVersionUID = 1L; @@ -251,6 +249,7 @@ public String toString() { } + private TaskIdentifier associatedTaskId; private SakerPath path; /** @@ -259,7 +258,8 @@ public String toString() { public LocalDirectoryClassFilesExecutionProperty() { } - public LocalDirectoryClassFilesExecutionProperty(SakerPath path) { + public LocalDirectoryClassFilesExecutionProperty(TaskIdentifier associatedTaskId, SakerPath path) { + this.associatedTaskId = associatedTaskId; this.path = path; } @@ -278,7 +278,7 @@ public PropertyValue getCurrentValue(ExecutionContext executioncontext) throws E } SakerPath cpabspath = path.resolve(keypath); ContentDescriptor classfilecd = executioncontext.getExecutionPropertyCurrentValue( - new LocalPathFileContentDescriptorExecutionProperty(cpabspath)); + new LocalPathFileContentDescriptorExecutionProperty(associatedTaskId, cpabspath)); if (classfilecd == null) { continue; } @@ -289,11 +289,13 @@ public PropertyValue getCurrentValue(ExecutionContext executioncontext) throws E @Override public void writeExternal(ObjectOutput out) throws IOException { + out.writeObject(associatedTaskId); out.writeObject(path); } @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + associatedTaskId = (TaskIdentifier) in.readObject(); path = (SakerPath) in.readObject(); } @@ -301,6 +303,7 @@ public void readExternal(ObjectInput in) throws IOException, ClassNotFoundExcept public int hashCode() { final int prime = 31; int result = 1; + result = prime * result + ((associatedTaskId == null) ? 0 : associatedTaskId.hashCode()); result = prime * result + ((path == null) ? 0 : path.hashCode()); return result; } @@ -314,6 +317,11 @@ public boolean equals(Object obj) { if (getClass() != obj.getClass()) return false; LocalDirectoryClassFilesExecutionProperty other = (LocalDirectoryClassFilesExecutionProperty) obj; + if (associatedTaskId == null) { + if (other.associatedTaskId != null) + return false; + } else if (!associatedTaskId.equals(other.associatedTaskId)) + return false; if (path == null) { if (other.path != null) return false; diff --git a/impl/src/main/saker/java/compiler/impl/compile/handler/full/FullCompilationHandler.java b/impl/src/main/saker/java/compiler/impl/compile/handler/full/FullCompilationHandler.java index 5bfd4f2..2073b08 100644 --- a/impl/src/main/saker/java/compiler/impl/compile/handler/full/FullCompilationHandler.java +++ b/impl/src/main/saker/java/compiler/impl/compile/handler/full/FullCompilationHandler.java @@ -370,8 +370,8 @@ public void visit(ExecutionFileLocation loc) { @Override public void visit(LocalFileLocation loc) { SakerPath path = loc.getLocalPath(); - taskContext.getTaskUtilities() - .getReportExecutionDependency(new LocalPathFileContentDescriptorExecutionProperty(path)); + taskContext.getTaskUtilities().getReportExecutionDependency( + new LocalPathFileContentDescriptorExecutionProperty(taskContext.getTaskId(), path)); paths.add(path); } }); diff --git a/impl/src/main/saker/java/compiler/impl/compile/handler/incremental/IncrementalCompilationHandler.java b/impl/src/main/saker/java/compiler/impl/compile/handler/incremental/IncrementalCompilationHandler.java index ea7d361..fb6370a 100644 --- a/impl/src/main/saker/java/compiler/impl/compile/handler/incremental/IncrementalCompilationHandler.java +++ b/impl/src/main/saker/java/compiler/impl/compile/handler/incremental/IncrementalCompilationHandler.java @@ -1177,7 +1177,7 @@ public void visit(ExecutionFileLocation loc) { public void visit(LocalFileLocation loc) { SakerPath path = loc.getLocalPath(); ContentDescriptor cd = taskContext.getTaskUtilities().getReportExecutionDependency( - new LocalPathFileContentDescriptorExecutionProperty(path)); + new LocalPathFileContentDescriptorExecutionProperty(taskContext.getTaskId(), path)); if (cd == null) { throw ObjectUtils.sneakyThrow( new FileNotFoundException("Module path local file not found: " + path)); @@ -1192,7 +1192,8 @@ public void visit(LocalFileLocation loc) { NavigableMap classfilecontents = taskContext .getTaskUtilities() - .getReportExecutionDependency(new LocalDirectoryClassFilesExecutionProperty(path)) + .getReportExecutionDependency(new LocalDirectoryClassFilesExecutionProperty( + taskContext.getTaskId(), path)) .getContents(); cpcontentdescriptors = classfilecontents; } else { @@ -1459,7 +1460,7 @@ public void visit(ExecutionFileLocation loc) { public void visit(LocalFileLocation loc) { SakerPath path = loc.getLocalPath(); ContentDescriptor cd = taskContext.getTaskUtilities().getReportExecutionDependency( - new LocalPathFileContentDescriptorExecutionProperty(path)); + new LocalPathFileContentDescriptorExecutionProperty(taskContext.getTaskId(), path)); if (cd == null) { throw ObjectUtils .sneakyThrow(new FileNotFoundException("Class path local file not found: " + path)); @@ -1474,7 +1475,8 @@ public void visit(LocalFileLocation loc) { NavigableMap classfilecontents = taskContext .getTaskUtilities() - .getReportExecutionDependency(new LocalDirectoryClassFilesExecutionProperty(path)) + .getReportExecutionDependency(new LocalDirectoryClassFilesExecutionProperty( + taskContext.getTaskId(), path)) .getContents(); cpcontentdescriptors = classfilecontents; } else { diff --git a/impl/src/main/saker/java/compiler/impl/compile/util/LocalPathFileContentDescriptorExecutionProperty.java b/impl/src/main/saker/java/compiler/impl/compile/util/LocalPathFileContentDescriptorExecutionProperty.java index 8c5faa8..5a9114d 100644 --- a/impl/src/main/saker/java/compiler/impl/compile/util/LocalPathFileContentDescriptorExecutionProperty.java +++ b/impl/src/main/saker/java/compiler/impl/compile/util/LocalPathFileContentDescriptorExecutionProperty.java @@ -25,11 +25,14 @@ import saker.build.file.provider.LocalFileProvider; import saker.build.runtime.execution.ExecutionContext; import saker.build.runtime.execution.ExecutionProperty; +import saker.build.task.identifier.TaskIdentifier; public class LocalPathFileContentDescriptorExecutionProperty implements ExecutionProperty, Externalizable { private static final long serialVersionUID = 1L; + //associatedTask field is necessary for the incremental behaviours of the build system to work + private TaskIdentifier associatedTask; private SakerPath path; /** @@ -38,7 +41,8 @@ public class LocalPathFileContentDescriptorExecutionProperty public LocalPathFileContentDescriptorExecutionProperty() { } - public LocalPathFileContentDescriptorExecutionProperty(SakerPath path) { + public LocalPathFileContentDescriptorExecutionProperty(TaskIdentifier associatedTask, SakerPath path) { + this.associatedTask = associatedTask; this.path = path; } @@ -55,11 +59,13 @@ public ContentDescriptor getCurrentValue(ExecutionContext executioncontext) { @Override public void writeExternal(ObjectOutput out) throws IOException { + out.writeObject(associatedTask); out.writeObject(path); } @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + associatedTask = (TaskIdentifier) in.readObject(); path = (SakerPath) in.readObject(); } @@ -67,6 +73,7 @@ public void readExternal(ObjectInput in) throws IOException, ClassNotFoundExcept public int hashCode() { final int prime = 31; int result = 1; + result = prime * result + ((associatedTask == null) ? 0 : associatedTask.hashCode()); result = prime * result + ((path == null) ? 0 : path.hashCode()); return result; } @@ -80,6 +87,11 @@ public boolean equals(Object obj) { if (getClass() != obj.getClass()) return false; LocalPathFileContentDescriptorExecutionProperty other = (LocalPathFileContentDescriptorExecutionProperty) obj; + if (associatedTask == null) { + if (other.associatedTask != null) + return false; + } else if (!associatedTask.equals(other.associatedTask)) + return false; if (path == null) { if (other.path != null) return false; diff --git a/impl/src/main/saker/java/compiler/impl/processor/ProcessorCreatorTaskFactory.java b/impl/src/main/saker/java/compiler/impl/processor/ProcessorCreatorTaskFactory.java index 5df8644..bd25489 100644 --- a/impl/src/main/saker/java/compiler/impl/processor/ProcessorCreatorTaskFactory.java +++ b/impl/src/main/saker/java/compiler/impl/processor/ProcessorCreatorTaskFactory.java @@ -216,6 +216,9 @@ public void visit(ExecutionFileLocation loc) { fp.writeToFile(in, temp); } }); + taskcontext.getTaskUtilities().getReportExecutionDependency( + new LocalPathFileContentDescriptorExecutionProperty(taskcontext.getTaskId(), + SakerPath.valueOf(cachejarpath))); } catch (IOException e) { throw ObjectUtils.sneakyThrow(e); } @@ -276,6 +279,9 @@ public void visit(LocalFileLocation loc) { fp.writeToFile(in, temp); } }); + taskcontext.getTaskUtilities().getReportExecutionDependency( + new LocalPathFileContentDescriptorExecutionProperty(taskcontext.getTaskId(), + SakerPath.valueOf(cachejarpath))); } catch (IOException e) { throw ObjectUtils.sneakyThrow(e); } @@ -290,6 +296,9 @@ public void visit(LocalFileLocation loc) { fp.writeToFile(in, temp); } }); + taskcontext.getTaskUtilities().getReportExecutionDependency( + new LocalPathFileContentDescriptorExecutionProperty(taskcontext.getTaskId(), + SakerPath.valueOf(cachejarpath))); } catch (NoSuchAlgorithmException | IOException e) { throw ObjectUtils.sneakyThrow(e); } @@ -509,7 +518,7 @@ public void visit(LocalFileLocation loc) { SakerPath path = loc.getLocalPath(); TaskExecutionUtilities taskutils = taskcontext.getTaskUtilities(); ContentDescriptor cd = taskutils.getReportExecutionDependency( - new LocalPathFileContentDescriptorExecutionProperty(path)); + new LocalPathFileContentDescriptorExecutionProperty(taskcontext.getTaskId(), path)); if (cd == null) { throw ObjectUtils .sneakyThrow(new FileNotFoundException("Class path local file not found: " + path)); @@ -520,7 +529,8 @@ public void visit(LocalFileLocation loc) { //add the dependencies on the files (all files, not only .class files) LocalDirectoryFilesExecutionProperty.PropertyValue pval = taskutils - .getReportExecutionDependency(new LocalDirectoryFilesExecutionProperty(path)); + .getReportExecutionDependency( + new LocalDirectoryFilesExecutionProperty(taskcontext.getTaskId(), path)); cdres[0] = new MultiPathContentDescriptor(pval.getContents()); localdircontentsconsumer.accept(loc, pval.getContents().navigableKeySet()); } else { @@ -655,6 +665,7 @@ public String toString() { } + private TaskIdentifier associatedTaskId; private SakerPath path; /** @@ -663,7 +674,8 @@ public String toString() { public LocalDirectoryFilesExecutionProperty() { } - public LocalDirectoryFilesExecutionProperty(SakerPath path) { + public LocalDirectoryFilesExecutionProperty(TaskIdentifier associatedTaskId, SakerPath path) { + this.associatedTaskId = associatedTaskId; this.path = path; } @@ -678,7 +690,7 @@ public PropertyValue getCurrentValue(ExecutionContext executioncontext) throws E SakerPath keypath = entry.getKey(); SakerPath cpabspath = path.resolve(keypath); ContentDescriptor classfilecd = executioncontext.getExecutionPropertyCurrentValue( - new LocalPathFileContentDescriptorExecutionProperty(cpabspath)); + new LocalPathFileContentDescriptorExecutionProperty(associatedTaskId, cpabspath)); if (classfilecd == null) { continue; } @@ -689,11 +701,13 @@ public PropertyValue getCurrentValue(ExecutionContext executioncontext) throws E @Override public void writeExternal(ObjectOutput out) throws IOException { + out.writeObject(associatedTaskId); out.writeObject(path); } @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException { + associatedTaskId = (TaskIdentifier) in.readObject(); path = (SakerPath) in.readObject(); } @@ -701,6 +715,7 @@ public void readExternal(ObjectInput in) throws IOException, ClassNotFoundExcept public int hashCode() { final int prime = 31; int result = 1; + result = prime * result + ((associatedTaskId == null) ? 0 : associatedTaskId.hashCode()); result = prime * result + ((path == null) ? 0 : path.hashCode()); return result; } @@ -714,6 +729,11 @@ public boolean equals(Object obj) { if (getClass() != obj.getClass()) return false; LocalDirectoryFilesExecutionProperty other = (LocalDirectoryFilesExecutionProperty) obj; + if (associatedTaskId == null) { + if (other.associatedTaskId != null) + return false; + } else if (!associatedTaskId.equals(other.associatedTaskId)) + return false; if (path == null) { if (other.path != null) return false;