Hierarchically scan directory structure for classfiles and matching files.

@param log
the log
16380
@Override
    void scanPaths(final LogNode log) {
        if (skipClasspathElement) {
            return;
        }
        if (scanned.getAndSet(true)) {
            // Should not happen
            throw new IllegalArgumentException("Already scanned classpath element " + toString());
        }

        final LogNode subLog = log == null ? null
                : log.log(classpathEltDir.getPath(), "Scanning directory classpath element " + classpathEltDir);

        scanDirRecursively(classpathEltDir, subLog);

        finishScanPaths(subLog);
    }
305357
@Override
    public void scanDirectoryNoResults(
            File directory, FileFilter fileFilter, Function<Resource, ?> fileProcessor) {
        this.scanDirectory(directory, null, fileFilter, fileProcessor);
    }
1222270
private void generateClassFiles(ClassTree classtree) {
        String[] packageNames = configuration.classDocCatalog.packageNames();
        for (String packageName : packageNames) {
            generateClassFiles(configuration.classDocCatalog.allClasses(
                    packageName), classtree);
        }
    }
1130227
public static <T> void initialize(final Log log, final List<Filter<T>> filters) {

        // Check sanity
        Validate.notNull(log, "log");
        Validate.notNull(filters, "filters");

        for (Filter<T> current : filters) {
            current.initialize(log);
        }
    }
513391
public static void infoAndDebug(Logger log, String methodName,
            Object... params) {
        if (!log.isInfoEnabled())
            return;
        int length = params.length;
        Object[] newParams = new Object[length];
        boolean hasCollection = false;
        for (int i = 0; i < length; i++) {
            if (params[i] instanceof Collection) {
                newParams[i] = (((Collection<?>) params[i]).size());
                hasCollection = true;
            } else
                newParams[i] = params[i];
        }
        if (hasCollection)
            log.info(buildMethodLogString(methodName, newParams));
        log.debug(buildMethodLogString(methodName, params));
    }
927295
public void setLogFileDir(String logDirName) {
        File logDir = new File(logDirName);
        if (!logDir.isAbsolute()) {
            logDir = new File(serverBaseDir, logDirName);
        }

        this.logFileDir = logDirName;
        if (started) {
            configuration.setLogFileDir(logDir.getAbsolutePath());
        }
    }
270189
public void scan(List<? extends JCTree> trees) {
        if (trees != null)
        for (List<? extends JCTree> l = trees; l.nonEmpty(); l = l.tail)
            scan(l.head);
    }


/* ***************************************************************************
 * Visitor methods
 ****************************************************************************/

    public void visitTopLevel(JCCompilationUnit tree) {
        scan(tree.packageAnnotations);
        scan(tree.pid);
        scan(tree.defs);
    }

    public void visitImport(JCImport tree) {
        scan(tree.qualid);
    }

    public void visitClassDef(JCClassDecl tree) {
        scan(tree.mods);
        scan(tree.typarams);
        scan(tree.extending);
        scan(tree.implementing);
        scan(tree.defs);
    }

    public void visitMethodDef(JCMethodDecl tree) {
        scan(tree.mods);
        scan(tree.restype);
        scan(tree.typarams);
        scan(tree.recvparam);
        scan(tree.params);
        scan(tree.thrown);
        scan(tree.defaultValue);
        scan(tree.body);
    }

    public void visitVarDef(JCVariableDecl tree) {
        scan(tree.mods);
        scan(tree.vartype);
        scan(tree.nameexpr);
        scan(tree.init);
    }

    public void visitSkip(JCSkip tree) {
    }
480728
static private List<File> getFileListingLogs(File aStartingDir)
            throws Exception {
        List<File> result = new ArrayList<File>();
        File[] logfiles = aStartingDir.listFiles(new FileFilter() {

            @Override
            public boolean accept(File pathname) {
                return pathname.isFile();
            }
        });
        List<File> logFilesList = Arrays.asList(logfiles);
        File[] allDirs = aStartingDir.listFiles(new FileFilter() {
            @Override
            public boolean accept(File pathname) {
                return pathname.isDirectory();
            }
        });
        for (File file : logFilesList) {
            if (file.getName().equals("log.xml")) {
                result.add(file);
            }
        }
        List<File> allDirsList = Arrays.asList(allDirs);
        Collections.sort(allDirsList, new Comparator<File>() {
            public int compare(File o1, File o2) {

                if (o1.lastModified() > o2.lastModified()) {
                    return +1;
                } else if (o1.lastModified() < o2.lastModified()) {
                    return -1;
                } else {
                    return 0;
                }
            }

        });
        for (File file : allDirsList) {
            if (!file.isFile()) {
                List<File> deeperList = getFileListingLogs(file);
                result.addAll(deeperList);
            }
        }
        return result;
    }
664321
private void openFile(final File log) throws IOException
   {
      SecurityHelper.doPrivilegedIOExceptionAction(new PrivilegedExceptionAction<Void>()
      {
         @SuppressWarnings("resource")
         public Void run() throws Exception
         {
            // set file size;
            if (!log.exists())
            {
               log.getParentFile().mkdirs();
               log.createNewFile();
               out = new FileOutputStream(log).getChannel();
               out.position(1024 * fileSize - 1);
               out.write(ByteBuffer.wrap(new byte[]{0}));
               out.position(0);
               out.force(false);
            }
            else
            {
               out = new FileOutputStream(log, true).getChannel();
            }
            return null;
         }
      });
   }
1063794
public void output(Logger log)
    {
        Map<String, Entry> entries = config.getSortedEntries();
        
        if ( entries.isEmpty() )
        {
            return;
        }

        ColumnPrinter printer = build(entries);

        log.debug("Configuration Details");
        for ( String line : printer.generate() )
        {
            log.debug(line);
        }
    }
1242844
@Deprecated
  public void setClassesDir(File classesDir) {
    if (classesDir == null) throw new NullPointerException("classesDir");
    getLogger().warn("The 'classesDir' property on the '{}' task is deprecated. Use 'classesDirs' of type FileCollection instead!",
        getName());
    setClassesDirs(getProject().files(classesDir));
  }
439661
@Override
    void open(final WorkQueue<ClasspathEntryWorkUnit> workQueue, final LogNode log) throws InterruptedException {
        if (!scanSpec.scanJars) {
            if (log != null) {
                log.log("Skipping classpath element, since jar scanning is disabled: " + rawPath);
            }
            skipClasspathElement = true;
            return;
        }
        final LogNode subLog = log == null ? null : log.log("Opening jar: " + rawPath);
        final int plingIdx = rawPath.indexOf('!');
        final String outermostZipFilePathResolved = FastPathResolver.resolve(FileUtils.CURR_DIR_PATH,
                plingIdx < 0 ? rawPath : rawPath.substring(0, plingIdx));
        if (!scanSpec.jarWhiteBlackList.isWhitelistedAndNotBlacklisted(outermostZipFilePathResolved)) {
            if (subLog != null) {
                subLog.log("Skipping jarfile that is blacklisted or not whitelisted: " + rawPath);
            }
            skipClasspathElement = true;
            return;
        }

        try {
            // Get LogicalZipFile for innermost nested jarfile
            Entry<LogicalZipFile, String> logicalZipFileAndPackageRoot;
            try {
                logicalZipFileAndPackageRoot = nestedJarHandler.nestedPathToLogicalZipFileAndPackageRootMap
                        .get(rawPath, subLog);
            } catch (final NullSingletonException e) {
                // Generally thrown on the second and subsequent attempt to call .get(), after the first failed
                throw new IOException("Could not get logical zipfile " + rawPath + " : " + e);
            }
            logicalZipFile = logicalZipFileAndPackageRoot.getKey();
            if (logicalZipFile == null) {
                // Should not happen, but this keeps lgtm static analysis happy
                throw new IOException("Logical zipfile was null");
            }

            // Get the normalized path of the logical zipfile
            zipFilePath = FastPathResolver.resolve(FileUtils.CURR_DIR_PATH, logicalZipFile.getPath());

            // Get package root of jarfile 
            final String packageRoot = logicalZipFileAndPackageRoot.getValue();
            if (!packageRoot.isEmpty()) {
                packageRootPrefix = packageRoot + "/";
            }
        } catch (final IOException | IllegalArgumentException e) {
            if (subLog != null) {
                subLog.log("Could not open jarfile " + rawPath + " : " + e);
            }
            skipClasspathElement = true;
            return;
        }

        if (!scanSpec.enableSystemJarsAndModules && logicalZipFile.isJREJar) {
            // Found a blacklisted JRE jar that was not caught by filtering for rt.jar in ClasspathFinder
            // (the isJREJar value was set by detecting JRE headers in the jar's manifest file)
            if (subLog != null) {
                subLog.log("Ignoring JRE jar: " + rawPath);
            }
            skipClasspathElement = true;
            return;
        }

        if (!logicalZipFile.isWhitelistedAndNotBlacklisted(scanSpec.jarWhiteBlackList)) {
            if (subLog != null) {
                subLog.log("Skipping jarfile that is blacklisted or not whitelisted: " + rawPath);
            }
            skipClasspathElement = true;
            return;
        }

        // Automatically add any nested "lib/" dirs to classpath, since not all classloaders return them
        // as classpath elements
        int childClasspathEntryIdx = 0;
        if (scanSpec.scanNestedJars) {
            for (final FastZipEntry zipEntry : logicalZipFile.entries) {
                for (final String libDirPrefix : ClassLoaderHandlerRegistry.AUTOMATIC_LIB_DIR_PREFIXES) {
                    if (zipEntry.entryNameUnversioned.startsWith(libDirPrefix)
                            && zipEntry.entryNameUnversioned.endsWith(".jar")) {
                        final String entryPath = zipEntry.getPath();
                        if (subLog != null) {
                            subLog.log("Found nested lib jar: " + entryPath);
                        }
                        workQueue.addWorkUnit(new ClasspathEntryWorkUnit(
                                /* rawClasspathEntry = */ new SimpleEntry<>(entryPath, classLoader),
                                /* parentClasspathElement = */ this,
                                /* orderWithinParentClasspathElement = */
                                childClasspathEntryIdx++));
                        break;
                    }
                }
            }
        }

        // Don't add child classpath elements that are identical to this classpath element, or that are duplicates
        final Set<String> scheduledChildClasspathElements = new HashSet<>();
        scheduledChildClasspathElements.add(rawPath);

        // Create child classpath elements from values obtained from Class-Path entry in manifest, resolving
        // the paths relative to the dir or parent jarfile that the jarfile is contained in
        if (logicalZipFile.classPathManifestEntryValue != null) {
            // Get parent dir of logical zipfile within grandparent slice,
            // e.g. for a zipfile slice path of "/path/to/jar1.jar!/lib/jar2.jar", this is "lib",
            // or for "/path/to/jar1.jar", this is "/path/to", or "" if the jar is in the toplevel dir.
            final String jarParentDir = FileUtils
                    .getParentDirPath(logicalZipFile.getPathWithinParentZipFileSlice());
            // Add paths in manifest file's "Class-Path" entry to the classpath, resolving paths relative to
            // the parent directory or jar
            for (final String childClassPathEltPathRelative : logicalZipFile.classPathManifestEntryValue
                    .split(" ")) {
                if (!childClassPathEltPathRelative.isEmpty()) {
                    // Resolve Class-Path entry relative to containing dir
                    String childClassPathEltPath = FastPathResolver.resolve(jarParentDir,
                            childClassPathEltPathRelative);
                    // If this is a nested jar, prepend outer jar prefix
                    final ZipFileSlice parentZipFileSlice = logicalZipFile.getParentZipFileSlice();
                    if (parentZipFileSlice != null) {
                        childClassPathEltPath = parentZipFileSlice.getPath()
                                + (childClassPathEltPath.startsWith("/") ? "!" : "!/") + childClassPathEltPath;
                    }
                    // Only add child classpath elements once
                    if (scheduledChildClasspathElements.add(childClassPathEltPath)) {
                        // Schedule child classpath element for scanning
                        workQueue.addWorkUnit( //
                                new ClasspathEntryWorkUnit(
                                        /* rawClasspathEntry = */ new SimpleEntry<>(childClassPathEltPath,
                                                classLoader),
                                        /* parentClasspathElement = */ this,
                                        /* orderWithinParentClasspathElement = */
                                        childClasspathEntryIdx++));
                    }
                }
            }
        }
        // Add paths in an OSGi bundle jar manifest's "Bundle-ClassPath" entry to the classpath, resolving 
        // the paths relative to the root of the jarfile
        if (logicalZipFile.bundleClassPathManifestEntryValue != null) {
            final String zipFilePathPrefix = zipFilePath + "!/";
            // Class-Path is split on " ", but Bundle-ClassPath is split on ","
            for (String childBundlePath : logicalZipFile.bundleClassPathManifestEntryValue.split(",")) {
                // Assume that Bundle-ClassPath paths have to be given relative to jarfile root
                while (childBundlePath.startsWith("/")) {
                    childBundlePath = childBundlePath.substring(1);
                }
                // Currently the position of "." relative to child classpath entries is ignored (the
                // Bundle-ClassPath path is treated as if "." is in the first position, since child
                // classpath entries are always added to the classpath after the parent classpath
                // entry that they were obtained from).
                if (!childBundlePath.isEmpty() && !childBundlePath.equals(".")) {
                    // Resolve Bundle-ClassPath entry within jar
                    final String childClassPathEltPath = zipFilePathPrefix
                            + FileUtils.sanitizeEntryPath(childBundlePath, /* removeInitialSlash = */ true);
                    // Only add child classpath elements once
                    if (scheduledChildClasspathElements.add(childClassPathEltPath)) {
                        // Schedule child classpath element for scanning
                        workQueue.addWorkUnit(new ClasspathEntryWorkUnit(
                                /* rawClasspathEntry = */ //
                                new SimpleEntry<>(childClassPathEltPath, classLoader),
                                /* parentClasspathElement = */ this,
                                /* orderWithinParentClasspathElement = */
                                childClasspathEntryIdx++));
                    }
                }
            }
        }
    }
949599
public static void info(Logger log, String methodName) {
        if (log.isInfoEnabled())
            log.info(buildMethodLogString(methodName));
    }
60001
private static List<String> getResourcesFromDirectory(final File directory, final Pattern searchPattern) {
        final List<String> resources = new ArrayList<>();

        // Filter only properties files
        final File[] fileList = directory.listFiles();

        if (fileList != null && fileList.length > 0) {
            // Iterate over each relevant file
            for (final File file : fileList) {
                // If the file is a directory process a recursive call to explorer the tree
                if (file.isDirectory()) {
                    resources.addAll(getResourcesFromDirectory(file, searchPattern));
                } else {
                    try {
                        checkResource(resources, searchPattern, file.getCanonicalPath());
                    } catch (final IOException e) {
                        LOGGER.log(BAD_CANONICAL_PATH, e);
                    }
                }
            }
        }
        return resources;
    }
870518
private void scanClasses() throws PersistenceUnitScannerException {
        try {
            for (URL url : urlSet) {
                final HashSet<ClassInfoType> citSet = new HashSet<ClassInfoType>();
                final String urlProtocol = url.getProtocol();
                if ("file".equalsIgnoreCase(urlProtocol)) {
                    // Protocol is "file", which either addresses a jar file or an exploded jar file
                    final Path taPath = Paths.get(url.toURI());
                    if (Files.isDirectory(taPath)) {
                        // Exploded Archive
                        citSet.addAll(processExplodedJarFormat(taPath));
                    } else {
                        // Unexploded Archive
                        citSet.addAll(processUnexplodedFile(taPath));
                    }
                } else if (url.toString().startsWith("jar:file")) {
                    citSet.addAll(processJarFileURL(url));
                } else {
                    // InputStream will be in jar format.
                    citSet.addAll(processJarFormatInputStreamURL(url));
                }

                processInnerClasses(citSet);
                scannedClassesMap.put(url, citSet);
            }
        } catch (Exception e) {
            FFDCFilter.processException(e, PersistenceUnitScanner.class.getName() + ".scanClasses", "118");
            throw new PersistenceUnitScannerException(e);
        }
    }
76818
private void scanDirectory(final File directory, final String relativePath, final ScanContext scanContext) {
        final List<File> children = listDirectoryChildren(directory, filter);
        for (File child : children) {
            final String fileName = child.getName();
            if (fileName.endsWith(DEPLOYED)) {
                final String deploymentName = fileName.substring(0, fileName.length() - DEPLOYED.length());
                DeploymentMarker deploymentMarker = deployed.get(deploymentName);
                if (deploymentMarker == null) {
                    scanContext.toRemove.remove(deploymentName);
                    removeExtraneousMarker(child, fileName);
                } else {
                    final File deploymentFile = new File(directory, deploymentName);
                    if (deploymentFile.exists()) {
                        scanContext.toRemove.remove(deploymentName);
                        if (deployed.get(deploymentName).lastModified != child.lastModified()) {
                            scanContext.scannerTasks.add(new RedeployTask(deploymentName, child.lastModified(), directory,
                                    !child.isDirectory()));
                        } else {
                            // AS7-784 check for undeploy or removal of the deployment via another management client
                            Boolean isDeployed = scanContext.registeredDeployments.get(deploymentName);
                            if (isDeployed == null || !isDeployed) {
                                // It was undeployed or removed; get rid of the .deployed marker and
                                // put down a .undeployed marker so we don't process this again
                                deployed.remove(deploymentName);
                                removeExtraneousMarker(child, fileName);
                                final File marker = new File(directory, deploymentName + UNDEPLOYED);
                                createMarkerFile(marker, deploymentName);
                                if (isDeployed == null) {
                                    DeploymentScannerLogger.ROOT_LOGGER.scannerDeploymentRemovedButNotByScanner(deploymentName, marker);
                                } else {
                                    DeploymentScannerLogger.ROOT_LOGGER.scannerDeploymentUndeployedButNotByScanner(deploymentName, marker);
                                }
                            }
                        }
                    } else {
                        boolean autoDeployable = deploymentMarker.archive ? autoDeployZip : autoDeployExploded;
                        if (!autoDeployable) {
                            // Don't undeploy but log a warn if this is exploded content
                            scanContext.toRemove.remove(deploymentName);
                            if (!deploymentMarker.archive) {
                                scanContext.prematureExplodedDeletions.add(deploymentName);
                            }
                        }
                        // else AS7-1240 -- content is gone, leave deploymentName in scanContext.toRemove to trigger undeploy
                    }
                }
            } else if (fileName.endsWith(DO_DEPLOY) || (fileName.endsWith(FAILED_DEPLOY) && firstScan)) {
                // AS7-2581 - attempt to redeploy failed deployments on restart.
                final String markerStatus = fileName.endsWith(DO_DEPLOY) ? DO_DEPLOY : FAILED_DEPLOY;
                final String deploymentName = fileName.substring(0, fileName.length() - markerStatus.length());

                if (FAILED_DEPLOY.equals(markerStatus)) {
                    if (!scanContext.firstScanDeployments.add(deploymentName)) {
                        continue;
                    }
                    ROOT_LOGGER.reattemptingFailedDeployment(deploymentName);
                }

                final File deploymentFile = new File(directory, deploymentName);
                if (!deploymentFile.exists()) {
                    scanContext.ignoredMissingDeployments.add(deploymentName);
                    continue;
                }
                long timestamp = getDeploymentTimestamp(deploymentFile);
                final String path = relativeTo == null ? deploymentFile.getAbsolutePath() : relativePath + deploymentName; // TODO:
                // sub-directories
                // in
                // the
                // deploymentDir
                final boolean archive = deploymentFile.isFile();
                addContentAddingTask(path, archive, deploymentName, deploymentFile, timestamp, scanContext);
            } else if (fileName.endsWith(FAILED_DEPLOY)) {
                final String deploymentName = fileName.substring(0, fileName.length() - FAILED_DEPLOY.length());
                scanContext.toRemove.remove(deploymentName);
                if (!deployed.containsKey(deploymentName) && !(new File(child.getParent(), deploymentName).exists())) {
                    removeExtraneousMarker(child, fileName);
                }
            } else if (isEEArchive(fileName)) {
                boolean autoDeployable = child.isDirectory() ? autoDeployExploded : autoDeployZip;
                if (autoDeployable) {
                    if (!isAutoDeployDisabled(child)) {
                        long timestamp = getDeploymentTimestamp(child);
                        synchronizeScannerStatus(scanContext, directory, fileName, timestamp);
                        if (isFailedOrUndeployed(scanContext, directory, fileName, timestamp) || scanContext.firstScanDeployments.contains(fileName)) {
                            continue;
                        }

                        DeploymentMarker marker = deployed.get(fileName);
                        if (marker == null || marker.lastModified != timestamp) {
                            try {
                                if (isZipComplete(child)) {
                                    final String path = relativeTo == null ? child.getAbsolutePath() : relativePath + fileName;
                                    final boolean archive = child.isFile();
                                    if(firstScan){
                                        scanContext.firstScanDeployments.add(fileName);
                                    }
                                    addContentAddingTask(path, archive, fileName, child, timestamp, scanContext);
                                } else {
                                    //we need to make sure that the file was not deleted while
                                    //the scanner was running
                                    if (child.exists()) {
                                        scanContext.incompleteFiles.put(child, new IncompleteDeploymentStatus(child, timestamp));
                                    }
                                }
                            } catch (NonScannableZipException e) {
                                // Track for possible logging in scan()
                                scanContext.nonscannable.put(child, new NonScannableStatus(e, timestamp));
                            }
                        }
                    }
                } else if (!deployed.containsKey(fileName) && !new File(fileName + DO_DEPLOY).exists()
                        && !new File(fileName + FAILED_DEPLOY).exists()) {
                    // Track for possible INFO logging of the need for a marker
                    scanContext.nonDeployable.add(fileName);
                }
            } else if (isXmlFile(fileName)) {
                if (autoDeployXml) {
                    if (!isAutoDeployDisabled(child)) {
                        long timestamp = getDeploymentTimestamp(child);
                        if (isFailedOrUndeployed(scanContext, directory, fileName, timestamp) || scanContext.firstScanDeployments.contains(fileName)) {
                            continue;
                        }

                        DeploymentMarker marker = deployed.get(fileName);
                        if (marker == null || marker.lastModified != timestamp) {
                            if (isXmlComplete(child)) {
                                final String path = relativeTo == null ? child.getAbsolutePath() : relativePath + fileName;
                                if(firstScan){
                                    scanContext.firstScanDeployments.add(fileName);
                                }
                                addContentAddingTask(path, true, fileName, child, timestamp, scanContext);
                            } else {
                                //we need to make sure that the file was not deleted while
                                //the scanner was running
                                if (child.exists()) {
                                    scanContext.incompleteFiles.put(child, new IncompleteDeploymentStatus(child, timestamp));
                                }
                            }
                        }
                    }
                } else if (!deployed.containsKey(fileName) && !new File(fileName + DO_DEPLOY).exists()
                        && !new File(fileName + FAILED_DEPLOY).exists()) {
                    // Track for possible INFO logging of the need for a marker
                    scanContext.nonDeployable.add(fileName);
                }
            } else if (fileName.endsWith(DEPLOYING) || fileName.endsWith(UNDEPLOYING)) {
                // These markers should not survive a scan
                removeExtraneousMarker(child, fileName);
            } else if (fileName.endsWith(PENDING)) {
                // Do some housekeeping if the referenced deployment is gone
                final String deploymentName = fileName.substring(0, fileName.length() - PENDING.length());
                File deployment = new File(child.getParent(), deploymentName);
                if (!deployment.exists()) {
                    removeExtraneousMarker(child, fileName);
                }
            } else if (child.isDirectory()) { // exploded deployments would have been caught by isEEArchive(fileName) above

                if (WEB_INF.equalsIgnoreCase(fileName) || META_INF.equalsIgnoreCase(fileName)) {
                    // Looks like someone unzipped an archive in the scanned dir
                    // Track for possible ERROR logging
                    scanContext.illegalDir.add(fileName);
                } else {
                    scanDirectory(child, relativePath + child.getName() + File.separator, scanContext);
                }
            }
        }
    }
1664
private File determineLogDirectory(final BatchOptions options, String operation) {
        File logDirectoryParent = options != null ? options.getLogDirectoryParent() : null;
        if (logDirectoryParent == null) {
            logDirectoryParent = Files.createTempDir();
        }
        File logDirectory = new File(logDirectoryParent, "data-" + operation + "-reports");
        try {
            logDirectory = logDirectory.getCanonicalFile();
            FileUtils.deleteDirectory(logDirectory);
        } catch (IOException e) {
            throw new RuntimeException(
                    "Failed to clean data-" + operation + " log directory: " + logDirectory, e);
        }
        logDirectory.mkdirs();
        return logDirectory;
    }