diff --git a/.github/scripts/BinariesListUpdates.java b/.github/scripts/BinariesListUpdates.java deleted file mode 100644 index b9c2dbd0bc94..000000000000 --- a/.github/scripts/BinariesListUpdates.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import java.util.concurrent.Semaphore; -import java.util.concurrent.atomic.LongAdder; -import java.util.stream.Stream; -import org.apache.maven.search.api.Record; -import org.apache.maven.search.api.SearchRequest; -import org.apache.maven.search.backend.smo.SmoSearchBackend; -import org.apache.maven.search.backend.smo.SmoSearchBackendFactory; -import org.apache.maven.artifact.versioning.ComparableVersion; - -import static org.apache.maven.search.api.MAVEN.ARTIFACT_ID; -import static org.apache.maven.search.api.MAVEN.CLASSIFIER; -import static org.apache.maven.search.api.MAVEN.GROUP_ID; -import static org.apache.maven.search.api.MAVEN.VERSION; -import static org.apache.maven.search.api.request.BooleanQuery.and; -import static org.apache.maven.search.api.request.FieldQuery.fieldQuery; - -/** - * Scans for binaries-list files and checks if newer versions of the declared dependencies exist. - * - * dependencies: - * 
org.apache.maven.indexer:search-backend-smo
- * 
org.apache.maven:maven-artifact
- * - * @author mbien - */ -public class BinariesListUpdates { - - private static final LongAdder updates = new LongAdder(); - private static final LongAdder checks = new LongAdder(); - private static final LongAdder skips = new LongAdder(); - - // java --class-path "lib/*" BinariesListUpdates.java /path/to/netbeans/project - public static void main(String[] args) throws IOException, InterruptedException { - - if (args.length != 1 || Files.notExists(Path.of(args[0]).resolve("README.md"))) { - throw new IllegalArgumentException("path to netbeans folder expected"); - } - - Path path = Path.of(args[0]); - try (Stream dependencyFiles = Files.find(path, 10, (p, a) -> p.getFileName().toString().equals("binaries-list")); - SmoSearchBackend backend = SmoSearchBackendFactory.createSmo()) { - dependencyFiles.sorted().forEach(p -> { - try { - checkDependencies(p, backend); - } catch (IOException | InterruptedException ex) { - throw new RuntimeException(ex); - } - }); - } - - System.out.println("checked " + checks.sum() + " dependencies, found " + updates.sum() + " updates, skipped " + skips.sum() + "." ); - } - - private static void checkDependencies(Path path, SmoSearchBackend backend) throws IOException, InterruptedException { - System.out.println(path); - try (Stream lines = Files.lines(path).parallel()) { - - // 321C614F85F1DEA6BB08C1817C60D53B7F3552FD org.fusesource.jansi:jansi:2.4.0 - lines.filter(l -> !l.startsWith("#")) - .filter(l -> l.length() > 40 && l.charAt(40) == ' ') - .map(l -> l.substring(40+1)) - .forEach(l -> { - - String[] comp = l.split("\\:"); - if (comp.length == 3 || comp.length == 4) { - String gid = comp[0].strip(); - String aid = comp[1].strip(); - String version = comp[2].strip(); - String classifier = comp.length == 4 ? comp[3].strip() : null; - try { - String gac; - String latest; - if (classifier == null) { - latest = queryLatestVersion(backend, gid, aid); - gac = String.join(":", gid, aid); - } else { - latest = queryLatestVersion(backend, gid, aid, classifier.split("@")[0]); - gac = String.join(":", gid, aid, classifier); - } - if (latest != null && !version.equals(latest)) { - System.out.printf(" %-50s %s -> %s\n", gac, version, latest); - updates.increment(); - } - } catch (IOException | InterruptedException ex) { - throw new RuntimeException(ex); - } - } else { - System.out.println(" skip: '"+l+"'"); - skips.increment(); - } - checks.increment(); - }); - } - System.out.println(); - } - - private static String queryLatestVersion(SmoSearchBackend backend, String gid, String aid) throws IOException, InterruptedException { - return queryLatestVersion(backend, new SearchRequest(and(fieldQuery(GROUP_ID, gid), fieldQuery(ARTIFACT_ID, aid)))); - } - - private static String queryLatestVersion(SmoSearchBackend backend, String gid, String aid, String classifier) throws IOException, InterruptedException { - return queryLatestVersion(backend, new SearchRequest(and(fieldQuery(GROUP_ID, gid), fieldQuery(ARTIFACT_ID, aid), fieldQuery(CLASSIFIER, classifier)))); - } - - // reduce concurrency level if needed - private final static Semaphore requests = new Semaphore(3); - - private static String queryLatestVersion(SmoSearchBackend backend, SearchRequest request) throws IOException, InterruptedException { - requests.acquire(); - try { - List results; - try { - results = backend.search(request).getPage(); - } catch (IOException ex) { - System.out.println("received exception: '" + ex.getMessage() + "', retry in 10s"); - Thread.sleep(10_000); - try { - results = backend.search(request).getPage(); - } catch(IOException ignore) { - throw ex; - } - } - return results.stream() - .map(r -> r.getValue(VERSION)) - .filter(v -> !v.contains("alpha") && !v.contains("beta")) - .filter(v -> !v.contains("M") && !v.contains("m") && !v.contains("B") && !v.contains("b") && !v.contains("ea") && !v.contains("RC")) - .limit(5) - .max((v1, v2) -> new ComparableVersion(v1).compareTo(new ComparableVersion(v2))) - .orElse(null); - } finally { - requests.release(); - } - } - -} diff --git a/.github/workflows/dependency-checks.yml b/.github/workflows/dependency-checks-mvn.yml similarity index 80% rename from .github/workflows/dependency-checks.yml rename to .github/workflows/dependency-checks-mvn.yml index 9dc9f2101077..7e9c33a179cd 100644 --- a/.github/workflows/dependency-checks.yml +++ b/.github/workflows/dependency-checks-mvn.yml @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -name: NetBeans Dependency Checks +name: NetBeans Dependency Checks MVN on: # pull_request: @@ -57,9 +57,13 @@ jobs: - name: Check Dependencies run: | - DEPS=org.apache.maven:maven-artifact:3.9.11,org.apache.maven.indexer:search-backend-smo:7.1.6 - mvn eu.maveniverse.maven.plugins:toolbox:0.13.7:gav-resolve-transitive -Dgav=$DEPS -DsinkSpec="flat(./lib)" + ant -quiet bootstrap + ant -quiet verify-libs-and-licenses + cd nbbuild/build/mavenpoms/ echo "
" >> $GITHUB_STEP_SUMMARY
-          java -cp "lib/*" .github/scripts/BinariesListUpdates.java ./ | tee -a $GITHUB_STEP_SUMMARY
+          mvn -B\
+           -Dorg.slf4j.simpleLogger.defaultLogLevel=warn\
+           -Dorg.slf4j.simpleLogger.log.eu.maveniverse.maven=info\
+           -DupToDate=true\
+           eu.maveniverse.maven.plugins:toolbox:0.15.17:libyear | tee -a $GITHUB_STEP_SUMMARY
           echo "
" >> $GITHUB_STEP_SUMMARY - rm -Rf lib diff --git a/nbbuild/antsrc/org/netbeans/nbbuild/extlibs/MavenCoordinate.java b/nbbuild/antsrc/org/netbeans/nbbuild/extlibs/MavenCoordinate.java index 43932958f35a..b79a692d3821 100644 --- a/nbbuild/antsrc/org/netbeans/nbbuild/extlibs/MavenCoordinate.java +++ b/nbbuild/antsrc/org/netbeans/nbbuild/extlibs/MavenCoordinate.java @@ -25,21 +25,7 @@ import java.net.URLDecoder; import java.net.URLEncoder; -class MavenCoordinate { - - private final String groupId; - private final String artifactId; - private final String version; - private final String extension; - private final String classifier; - - private MavenCoordinate(String groupId, String artifactId, String version, String extension, String classifier) { - this.groupId = groupId; - this.artifactId = artifactId; - this.version = version; - this.extension = extension; - this.classifier = classifier; - } +record MavenCoordinate(String groupId, String artifactId, String version, String extension, String classifier) { public boolean hasClassifier() { return !classifier.isEmpty(); diff --git a/nbbuild/antsrc/org/netbeans/nbbuild/extlibs/MavenSkeletonProject.java b/nbbuild/antsrc/org/netbeans/nbbuild/extlibs/MavenSkeletonProject.java index 4a9cb50087f9..ca30026cb0eb 100644 --- a/nbbuild/antsrc/org/netbeans/nbbuild/extlibs/MavenSkeletonProject.java +++ b/nbbuild/antsrc/org/netbeans/nbbuild/extlibs/MavenSkeletonProject.java @@ -29,6 +29,9 @@ import java.nio.file.StandardOpenOption; import java.util.Arrays; import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; import java.util.Set; import java.util.TreeSet; import java.util.stream.Collectors; @@ -97,34 +100,47 @@ private void buildLibPomForMaven() throws IOException { } Path pseudoMavendirectory = Files.createDirectory(pseudoMaven); Path parentPom = Files.createFile(pseudoMavendirectory.resolve("pom.xml")); - Files.write(parentPom, ("" - + "\n 4.0.0" - + "\n com.mycompany.app" - + "\n my-app" - + "\n 1" - + "\n pom" - + "\n" - + " dummyhttps://netbeans.apache.org/dummydummy\n" - + " ").getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND); + Files.write(parentPom, (""" + + 4.0.0 + pseudo.org.netbeans + netbeans + 1 + pom + dummyhttps://netbeans.apache.org/dummydummy + """).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND); Files.write(parentPom, "\n ".getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); + // scan module with external folder + Map> clusters = new HashMap<>(); for (String module : modules) { File d = new File(new File(nball, module), "external"); if (d.exists() && d.isDirectory()) { - String moduleName = module.replace("/", "").replace(".", ""); - Files.write(parentPom, ("\n " + moduleName + "").getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); - Path modulesPathFolder = Files.createDirectory(pseudoMavendirectory.resolve(moduleName)); - Path moduleparentPom = Files.createFile(modulesPathFolder.resolve("pom.xml")); - Files.write(moduleparentPom, ("" - + "\n 4.0.0" - + "\n com.mycompany.appmy-app1" - + "\n com.mycompany.app" - + "\n " + moduleName + "" - + "\n 1").getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND); + var name = module.split("/"); + String clusterName = module.split("/")[0]; + clusters.computeIfAbsent(clusterName,k-> new HashSet<>()); + Path clusterfolder;String moduleName; + if (name.length > 1) { + moduleName = name[1]; + clusters.get(name[0]).add(moduleName); + } else { + moduleName = "nbbuild"; + } + clusterfolder = Files.createDirectories(pseudoMavendirectory.resolve(clusterName).resolve(moduleName)); + // write pom for the module in clusterfolder/modulefolder/pom.xml + Path moduleparentPom = Files.createFile(clusterfolder.resolve("pom.xml")); + Files.write(moduleparentPom, (""" + + 4.0.0 + pseudo.org.netbeans%s1 + %s + %s + 1""".formatted(clusterName,clusterName,moduleName)).getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND); Files.write(moduleparentPom, "\n ".getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); File list = new File(d, "binaries-list"); if (list.isFile()) { - + // do not accept more than once a g:a artifacts + Set duplicatecheck = new HashSet<>(); try ( Reader r = new FileReader(list)) { BufferedReader br = new BufferedReader(r); String line; @@ -139,18 +155,21 @@ private void buildLibPomForMaven() throws IOException { if (hashAndFile.length < 2) { throw new BuildException("Bad line '" + line + "' in " + list); } + if (MavenCoordinate.isMavenFile(hashAndFile[1])) { MavenCoordinate coordinate = MavenCoordinate.fromGradleFormat(hashAndFile[1]); - Files.write(moduleparentPom, ("\n ").getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); - Files.write(moduleparentPom, ("\n " + coordinate.getGroupId() + "").getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); - Files.write(moduleparentPom, ("\n " + coordinate.getArtifactId() + "").getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); - Files.write(moduleparentPom, ("\n " + coordinate.getVersion() + "").getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); - Files.write(moduleparentPom, ("\n " + coordinate.getExtension() + "").getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); - - if (coordinate.hasClassifier()) { - Files.write(moduleparentPom, ("\n " + coordinate.getClassifier() + "").getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); + if (duplicatecheck.add(MavenGA.from(coordinate))) { + Files.write(moduleparentPom, ("\n ").getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); + Files.write(moduleparentPom, ("\n " + coordinate.getGroupId() + "").getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); + Files.write(moduleparentPom, ("\n " + coordinate.getArtifactId() + "").getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); + Files.write(moduleparentPom, ("\n " + coordinate.getVersion() + "").getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); + Files.write(moduleparentPom, ("\n " + coordinate.getExtension() + "").getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); + + if (coordinate.hasClassifier()) { + Files.write(moduleparentPom, ("\n " + coordinate.getClassifier() + "").getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); + } + Files.write(moduleparentPom, ("\n ").getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); } - Files.write(moduleparentPom, ("\n ").getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); } } } @@ -160,53 +179,73 @@ private void buildLibPomForMaven() throws IOException { } } + // write clusters pom + for (Map.Entry> clusterEntry : clusters.entrySet()) { + String clusterName = clusterEntry.getKey(); + Path clusterFolder = pseudoMavendirectory.resolve(clusterName) ; + Files.write(parentPom, ("\n " + clusterName + "").getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); + + Path clusterparentPom = Files.createFile(clusterFolder.resolve("pom.xml")); + Files.write(clusterparentPom, (""" + + 4.0.0 + pseudo.org.netbeansnetbeans1 + """ + clusterName + "pom").getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND); + if (!clusterEntry.getValue().isEmpty()) { + Files.write(clusterparentPom, "\n ".getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); + for (String module : clusterEntry.getValue()) { + Files.write(clusterparentPom, ("\n " + module + "").getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); + + } + Files.write(clusterparentPom, "\n ".getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); + } + Files.write(clusterparentPom, "".getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); + } + // parent pom with plugin for report owasp Files.write(parentPom, "\n ".getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); - Files.write(parentPom, ("\n \n" - + " \n" - + " \n" - + " \n" - + " maven-site-plugin\n" - + " 4.0.0-M1\n" - + " \n" - + " \n" - + " \n" - + " ").getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); - - Files.write(parentPom, ("\n \n" - + " \n" - + " \n" - + " org.owasp\n" - + " dependency-check-maven\n" - + " 7.1.0\n" - + " \n" - + " false\n" - + " false\n" - + " \n" - + " \n" - + " \n" - + " \n" - + " aggregate\n" - + " \n" - + " \n" - + " \n" - + " \n" - + " \n" - + " \n" - + " org.codehaus.mojo\n" - + " versions-maven-plugin\n" - + " 2.11.0\n" - + " \n" - + " \n" - + " \n" - + " dependency-updates-report\n" - + " \n" - + " \n" - + " \n" - + " \n" - + " \n" - + " ").getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); + Files.write(parentPom, (""" + + + + + maven-site-plugin + 3.22.0 + + + + """).getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); + + Files.write(parentPom, (""" + + + + org.owasp + dependency-check-maven + 13.0.0 + + false + false + + + + + aggregate + + + + + + """).getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); Files.write(parentPom, "".getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); } + record MavenGA(String groupId, String artifactId) { + + private static MavenGA from(MavenCoordinate coordinate) { + return new MavenGA(coordinate.getGroupId(), coordinate.getArtifactId()); + } + + } + } diff --git a/nbbuild/build.properties b/nbbuild/build.properties index 2311f404f16c..692f477ba56f 100644 --- a/nbbuild/build.properties +++ b/nbbuild/build.properties @@ -22,7 +22,7 @@ test.user.dir=testuserdir nb.run.validation=true build.compiler.debug=on -bootstrap.jdk.release=11 +bootstrap.jdk.release=17 # Options to pass to NetBeans when starting it with "ant tryme": tryme.arg.hack=-J-Dnetbeans.full.hack=true diff --git a/nbbuild/nbproject/project.xml b/nbbuild/nbproject/project.xml index 976d1a4e0fe3..1d8ab99c6bbf 100644 --- a/nbbuild/nbproject/project.xml +++ b/nbbuild/nbproject/project.xml @@ -211,14 +211,14 @@ ${ant.core.lib}:${nb_all}/platform/javahelp/external/jhall-2.0_05.jar:${nb_all}/nbbuild/external/json-simple-1.1.1.jar:${nb_all}/nbbuild/external/jsoup-1.15.3.jar ${nb.build.dir}/antclasses ${nbantext.jar} - 11 + 17 test/unit/src ${test.unit.cp} ${nb.build.dir}/test/unit/classes - 11 + 17