# HG changeset patch # User Tim Boudreau # Date 1472884896 14400 # Node ID 04a79821e760e3fd16f42f2e22fa548bd693a501 # Parent 3527a32a19f05b871af2841f6c605d48699a16f8 Eliminate duplicates in graph files diff -r 3527a32a19f0 -r 04a79821e760 callgraph/pom.xml --- a/callgraph/pom.xml Mon Aug 29 13:00:53 2016 -0400 +++ b/callgraph/pom.xml Sat Sep 03 02:41:36 2016 -0400 @@ -4,7 +4,7 @@ org.netbeans callgraph Callgraph Utility - 1.1 + 1.3 jar UTF-8 diff -r 3527a32a19f0 -r 04a79821e760 callgraph/src/main/java/org/netbeans/lib/callgraph/Arguments.java --- a/callgraph/src/main/java/org/netbeans/lib/callgraph/Arguments.java Mon Aug 29 13:00:53 2016 -0400 +++ b/callgraph/src/main/java/org/netbeans/lib/callgraph/Arguments.java Sat Sep 03 02:41:36 2016 -0400 @@ -63,6 +63,8 @@ private static final Command[] commands = new Command[]{ new NoSelfReferencesCommand(), new ShortNamesCommand(), + new ExtendedPropertiesCommand(), + new AntCommand(), new MavenCommand(), new GradleCommand(), new IgnoreCommand(), @@ -75,12 +77,16 @@ new OmitAbstractCommand(), new QuietCommand(), new ReverseCommand(), + new AggressiveCommand(), new VerboseCommand() }; private boolean noSelfReferences = false; private boolean shortNames = false; private boolean maven = false; private boolean gradle = false; + private boolean ant = false; + private boolean xprop = false; + private boolean aggressive = false; private File outfile; private Set exclude = new HashSet<>(); private Set ignore = new HashSet<>(); @@ -141,13 +147,14 @@ } } } - Set origFolders = folders; - if (maven && gradle) { - errors.add("--maven and --gradle are mutually exclusive"); + if ((maven && gradle) || (ant && gradle) || (ant && maven)) { + errors.add("--maven, --ant and --gradle are mutually exclusive"); } else if (maven) { findMavenSubfolders(errors); } else if (gradle) { findGradleSubfolders(errors); + } else if (ant) { + findAntSubfolders(errors); } Set toIgnore = new HashSet<>(); for (String ig : ignore) { @@ -193,6 +200,12 @@ } } this.folders.removeAll(toIgnore); + if (verbose && !this.folders.isEmpty()) { + System.err.println("Will scan the following source roots:"); + for (File f : folders()) { + System.err.println(" " + f.getAbsolutePath()); + } + } if (packageGraphFile != null) { File parent = packageGraphFile.getParentFile(); @@ -250,6 +263,36 @@ return pom.exists() && pom.isFile() && pom.canRead(); } + void findAntSubfolders(List errors) { + Set flds = new HashSet<>(this.folders); + this.folders.clear(); + for (File f : flds) { + recurseSubfoldersLookingForAntProjects(f); + } + if (this.folders.isEmpty()) { + errors.add("Did not find any ant projects (looked for build.xml and src/ in all subfolders of folder list)"); + } + } + + private void recurseSubfoldersLookingForAntProjects(File file) { + if (file.isDirectory()) { + if (hasBuildXml(file)) { + File sources = new File(file, "src"); + if (sources.exists() && sources.isDirectory()) { + this.folders.add(sources); + } + } + for (File child : file.listFiles()) { + recurseSubfoldersLookingForAntProjects(child); + } + } + } + + private boolean hasBuildXml(File fld) { + File pom = new File(fld, "build.xml"); + return pom.exists() && pom.isFile() && pom.canRead(); + } + void findGradleSubfolders(List errors) { Set flds = new HashSet<>(this.folders); this.folders.clear(); @@ -333,6 +376,20 @@ } @Override + public boolean isAnt() { + return ant; + } + + @Override + public boolean isExtendedProperties() { + return xprop; + } + + public boolean isAggressive() { + return aggressive; + } + + @Override public File classGraphFile() { return classGraphFile; } @@ -467,7 +524,7 @@ @Override protected String help() { - return "Disable string memory optimizations - runs faster but may run out of memory"; + return "Disable string memory optimizations - runs faster and supports unicode class names, but may run out of memory"; } } @@ -489,6 +546,25 @@ } } + private static final class AggressiveCommand extends Command { + + AggressiveCommand() { + super(CMD_AGGRESSIVE_MEMORY, "z", true, false); + } + + @Override + protected int doParse(int i, String[] args, Arguments toSet) { + toSet.aggressive = true; + return 1; + } + + @Override + protected String help() { + return "Aggressively optimize the 8-bit string intern table " + + "for large graphs, sacrificing performace for space"; + } + } + private static final class ShortNamesCommand extends Command { ShortNamesCommand() { @@ -507,6 +583,42 @@ } } + private static final class ExtendedPropertiesCommand extends Command { + + ExtendedPropertiesCommand() { + super(CMD_EXTENDED_PROPERTIES, "x", true, false); + } + + @Override + protected int doParse(int i, String[] args, Arguments toSet) { + toSet.xprop = true; + return 1; + } + + @Override + protected String help() { + return "Find all maven projects that are children of the passed folders, and scan their src/main/java subfolders"; + } + } + + private static final class AntCommand extends Command { + + AntCommand() { + super(CMD_ANT, "a", true, false); + } + + @Override + protected int doParse(int i, String[] args, Arguments toSet) { + toSet.ant = true; + return 1; + } + + @Override + protected String help() { + return "Find all ant projects that are children of the passed folders, and scan their src/ subfolders"; + } + } + private static final class MavenCommand extends Command { MavenCommand() { diff -r 3527a32a19f0 -r 04a79821e760 callgraph/src/main/java/org/netbeans/lib/callgraph/Callgraph.java --- a/callgraph/src/main/java/org/netbeans/lib/callgraph/Callgraph.java Mon Aug 29 13:00:53 2016 -0400 +++ b/callgraph/src/main/java/org/netbeans/lib/callgraph/Callgraph.java Sat Sep 03 02:41:36 2016 -0400 @@ -1,7 +1,7 @@ /* * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. * - * Copyright (C) 1997-2015 Oracle and/or its affiliates. All rights reserved. + * Copyright (C) 1997-2016 Oracle and/or its affiliates. All rights reserved. * * Oracle and Java are registered trademarks of Oracle and/or its affiliates. * Other names may be trademarks of their respective owners. @@ -70,10 +70,8 @@ import java.util.LinkedList; import java.util.List; import java.util.Set; -import java.util.TreeSet; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; -import org.netbeans.lib.callgraph.util.ComparableCharSequence; /** * Scans a source folder and runs javac against any Java sources present, and @@ -134,7 +132,7 @@ * @throws IOException If i/o fails */ static List invoke(CallgraphControl arguments, Listener listener) throws IOException { - SourcesInfo info = new SourcesInfo(arguments.isDisableEightBitStrings()); + SourcesInfo info = new SourcesInfo(arguments.isDisableEightBitStrings(), arguments.isAggressive()); // Build an iterable of all Java sources (without collecting them all ahead of time) List> iterables = new LinkedList<>(); for (File folder : arguments) { @@ -158,8 +156,15 @@ // duplicate avoidance Set emittedPackageLines = new HashSet<>(); Set emittedClassLines = new HashSet<>(); + List clazz = new ArrayList<>(5); + CharSequence lastClass = null; + + List pkg = new ArrayList<>(5); + CharSequence lastPackage = null; + try { // Iterate every method + outer: for (SourceElement sce : all) { if (arguments.isExcluded(sce.qname().toString())) { // Ignore matches continue; @@ -167,7 +172,32 @@ List outbounds = new ArrayList<>(arguments.isReverse() ? sce.getInboundReferences() : sce.getOutboundReferences()); Collections.sort(outbounds); // also sort connections // Iterate the current method's connections - Set mth = new TreeSet<>(); + List mth = new ArrayList<>(5); + CharSequence currClazz = arguments.isShortNames() ? sce.typeName() : info.strings.concat(sce.packageName(), info.strings.DOT, sce.typeName()); + if (!currClazz.equals(lastClass)) { + if (classStream != null) { + writeLine(clazz, info, emittedClassLines, classStream); + } + clazz.clear(); + lastClass = currClazz; + } + if (clazz.isEmpty()) { + clazz.add(currClazz); + if (arguments.isExtendedProperties()) { + clazz.add(sce.isAbstract()); + } + } + CharSequence currPkg = sce.packageName(); + if (pkg.isEmpty()) { + pkg.add(currPkg); + } + if (!currPkg.equals(lastPackage)) { + if (packageStream != null) { + writeLine(pkg, info, emittedPackageLines, packageStream); + } + lastPackage = currPkg; + pkg.clear(); + } for (SourceElement outbound : outbounds) { if (arguments.isExcluded(outbound.qname().toString())) { // Ignore matches continue; @@ -181,24 +211,20 @@ if (!arguments.isSelfReferences() && sce.typeName().equals(outbound.typeName())) { continue; } - if (outStream != null) { + if (outStream != null || !arguments.isQuiet()) { if (arguments.isShortNames()) { mth.add(outbound.shortName()); } else { mth.add(outbound.qname()); } + if (arguments.isExtendedProperties()) { + mth.add(outbound.isAbstract()); + } } // Build the package graph output if necessary if (packageStream != null) { - CharSequence pkg1 = sce.packageName(); - CharSequence pkg2 = outbound.packageName(); - if (!pkg1.equals(pkg2)) { -// CharSequence pkgLine = '"' + pkg1.toString() + "\" \"" + pkg2.toString() + '"'; - CharSequence pkgLine = info.strings.concat(info.strings.QUOTE, pkg1, info.strings.CLOSE_OPEN_QUOTE, pkg2, info.strings.QUOTE); - if (!emittedPackageLines.contains(pkgLine)) { - emittedPackageLines.add(pkgLine); - packageStream.println(pkgLine); - } + if (!outbound.packageName().equals(currPkg) || arguments.isSelfReferences()) { + pkg.add(outbound.packageName()); } } // Build the class graph output if necessary @@ -206,22 +232,24 @@ CharSequence type1 = sce.typeName(); CharSequence type2 = outbound.typeName(); if (!arguments.isShortNames()) { - type1 = info.strings.concat(sce.packageName(), info.strings.DOT, type1); +// type1 = info.strings.concat(sce.packageName(), info.strings.DOT, type1); type2 = info.strings.concat(outbound.packageName(), info.strings.DOT, type2); } - if (!type1.equals(type2)) { - CharSequence classLine = info.strings.concat(info.strings.QUOTE, type1, info.strings.CLOSE_OPEN_QUOTE, type2, info.strings.QUOTE); - if (!emittedClassLines.contains(classLine)) { - emittedClassLines.add(classLine); - classStream.println(classLine); + if (!type1.equals(type2) && !clazz.contains(type2)) { + clazz.add(type2); + if (arguments.isExtendedProperties()) { + clazz.add(outbound.isAbstract()); } } } } - if ((!arguments.isQuiet() || outStream != null)) { + if (!arguments.isQuiet() || outStream != null) { if (!mth.isEmpty()) { CharSequence nm = arguments.isShortNames() ? sce.shortName() : sce.qname(); - List l = new ArrayList<>(mth); + List l = mth; + if (arguments.isExtendedProperties()) { + mth.add(0, sce.isAbstract()); + } l.add(0, nm); CharSequence line = info.strings.concatQuoted(l); if (!arguments.isQuiet()) { @@ -233,6 +261,12 @@ } } } + if (classStream != null && !clazz.isEmpty()) { + writeLine(clazz, info, emittedClassLines, classStream); + } + if (packageStream != null && !pkg.isEmpty()) { + writeLine(pkg, info, emittedPackageLines, packageStream); + } } finally { for (PrintStream ps : new PrintStream[]{outStream, packageStream, classStream}) { if (ps != null) { @@ -243,6 +277,15 @@ } return all; } + private static void writeLine(List clazz, SourcesInfo info, Set emittedClassLines, PrintStream classStream) { + if (!clazz.isEmpty()) { + CharSequence cs = info.strings.concatQuoted(clazz); + if (!emittedClassLines.contains(cs)) { + classStream.println(cs); + emittedClassLines.add(cs); + } + } + } private static PrintStream createPrintStreamIfNotNull(File outputFile) throws IOException { PrintStream outStream = null; diff -r 3527a32a19f0 -r 04a79821e760 callgraph/src/main/java/org/netbeans/lib/callgraph/CallgraphControl.java --- a/callgraph/src/main/java/org/netbeans/lib/callgraph/CallgraphControl.java Mon Aug 29 13:00:53 2016 -0400 +++ b/callgraph/src/main/java/org/netbeans/lib/callgraph/CallgraphControl.java Sat Sep 03 02:41:36 2016 -0400 @@ -1,4 +1,4 @@ -/* +/* * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. * * Copyright (C) 1997-2015 Oracle and/or its affiliates. All rights reserved. @@ -57,8 +57,11 @@ static final String CMD_NOSELF = "noself"; static final String CMD_SIMPLE = "simple"; + static final String CMD_ANT = "ant"; static final String CMD_MAVEN = "maven"; static final String CMD_GRADLE = "gradle"; + static final String CMD_EXTENDED_PROPERTIES = "extensions"; + static final String CMD_AGGRESSIVE_MEMORY = "aggressive"; static final String CMD_IGNORE = "ignore"; static final String CMD_PACKAGEGRAPH = "packagegraph"; static final String CMD_METHODGRAPH = "methodgraph"; @@ -80,6 +83,14 @@ boolean isMaven(); + boolean isGradle(); + + boolean isAnt(); + + boolean isExtendedProperties(); + + boolean isAggressive(); + boolean isQuiet(); boolean isSelfReferences(); diff -r 3527a32a19f0 -r 04a79821e760 callgraph/src/main/java/org/netbeans/lib/callgraph/javac/SourcesInfo.java --- a/callgraph/src/main/java/org/netbeans/lib/callgraph/javac/SourcesInfo.java Mon Aug 29 13:00:53 2016 -0400 +++ b/callgraph/src/main/java/org/netbeans/lib/callgraph/javac/SourcesInfo.java Sat Sep 03 02:41:36 2016 -0400 @@ -75,8 +75,8 @@ public final EightBitStrings strings; - public SourcesInfo(boolean eightBitStringsDisabled) { - this.strings = new EightBitStrings(eightBitStringsDisabled); + public SourcesInfo(boolean eightBitStringsDisabled, boolean aggressive) { + this.strings = new EightBitStrings(eightBitStringsDisabled, aggressive); } // XXX rewrite to not hold references to ClassTree, just names, diff -r 3527a32a19f0 -r 04a79821e760 callgraph/src/main/java/org/netbeans/lib/callgraph/util/EightBitStrings.java --- a/callgraph/src/main/java/org/netbeans/lib/callgraph/util/EightBitStrings.java Mon Aug 29 13:00:53 2016 -0400 +++ b/callgraph/src/main/java/org/netbeans/lib/callgraph/util/EightBitStrings.java Sat Sep 03 02:41:36 2016 -0400 @@ -70,14 +70,22 @@ private final InternTable INTERN_TABLE = new InternTable(); public final CharSequence DOT = create("."); public final CharSequence QUOTE = create("\""); + public final CharSequence SPACE = create(" "); + public final CharSequence QUOTE_SPACE = create("\" "); public final CharSequence CLOSE_OPEN_QUOTE = create("\" \""); - private boolean disabled; + private final boolean disabled; + private final boolean aggressive; public EightBitStrings(boolean disabled) { - this.disabled = disabled; + this (disabled, false); } + public EightBitStrings(boolean disabled, boolean aggressive) { + this.disabled = disabled; + this.aggressive = aggressive; + } + public void clear() { INTERN_TABLE.dispose(); } @@ -87,6 +95,9 @@ return string instanceof ComparableCharSequence ? (ComparableCharSequence) string : new StringWrapper(string.toString()); } + if (aggressive) { + return concat(string); + } return INTERN_TABLE.intern(string); } @@ -100,32 +111,66 @@ sb.append(c); } return new StringWrapper(sb.toString()); + } else if (aggressive) { + List nue = new ArrayList<>(seqs.length + (seqs.length / 2)); + for (CharSequence seq : seqs) { + int ln = seq.length(); + StringBuilder sb = new StringBuilder(); + for(int i=0; i < ln; i++) { + char c = seq.charAt(i); + if (Character.isLetter(c) || Character.isDigit(c)) { + sb.append(c); + } else { + nue.add(sb.toString()); + sb = new StringBuilder(); + nue.add(new String(new char[] { c })); + } + } + if (sb.length() > 0) { + nue.add(sb.toString()); + } + } + if (nue.size() != seqs.length) { + seqs = nue.toArray(new CharSequence[nue.size()]); + } } return new Concatenation(seqs); } - public ComparableCharSequence concatQuoted(Collection seqs) { + public ComparableCharSequence concatQuoted(Collection seqs) { if (disabled) { StringBuilder sb = new StringBuilder("\""); - for (Iterator it = seqs.iterator(); it.hasNext();) { - CharSequence c = it.next(); + boolean first = true; + for (Iterator it = seqs.iterator(); it.hasNext();) { + Object c = it.next(); + if (!first) { + sb.append(SPACE); + } + if (c instanceof CharSequence) { + sb.append(QUOTE); + } sb.append(c); - if (it.hasNext()) { - sb.append("\" \""); - } else { - sb.append("\""); + if (c instanceof CharSequence) { + sb.append(QUOTE); } + first = false; } return new StringWrapper(sb.toString()); } else { - List quoted = new ArrayList<>((seqs.size() * 2) + 1); - quoted.add(this.QUOTE); - for (Iterator it = seqs.iterator(); it.hasNext();) { - quoted.add(it.next()); - if (it.hasNext()) { - quoted.add(CLOSE_OPEN_QUOTE); + List quoted = new ArrayList<>((seqs.size() * 3) + 1); + for (Iterator it = seqs.iterator(); it.hasNext();) { + Object c = it.next(); + if (c instanceof CharSequence) { + quoted.add(QUOTE); + quoted.add((CharSequence)c); + if (it.hasNext()) { + quoted.add(QUOTE_SPACE); + } else { + quoted.add(QUOTE); + } } else { - quoted.add(QUOTE); + quoted.add(create(c.toString())); + quoted.add(SPACE); } } return new Concatenation(quoted.toArray(new CharSequence[quoted.size()])); @@ -153,6 +198,10 @@ int internTableSize() { return INTERN_TABLE.last + 1; } + + List dumpInternTable() { + return INTERN_TABLE.dumpInternTable(); + } static class InternTable { @@ -187,6 +236,10 @@ Arrays.sort(entries, 0, last + 1); return entry; } + + List dumpInternTable() { + return Arrays.asList(entries); + } private static final class Entry implements ComparableCharSequence { @@ -316,10 +369,16 @@ private final InternTable.Entry[] entries; Concatenation(CharSequence... entries) { - this.entries = new InternTable.Entry[entries.length]; - for (int i = 0; i < entries.length; i++) { - this.entries[i] = INTERN_TABLE.intern(entries[i]); + List l = new ArrayList<>(entries.length); + for (CharSequence cs : entries) { + if (cs instanceof Concatenation) { + Concatenation c1 = (Concatenation) cs; + l.addAll(Arrays.asList(c1.entries)); + } else { + l.add(INTERN_TABLE.intern(cs)); + } } + this.entries = l.toArray(new InternTable.Entry[l.size()]); } @Override @@ -358,7 +417,7 @@ sb.append(e); } if (debug) { - sb.append (" - " + Arrays.asList(entries)); + sb.append(" - ").append (Arrays.asList(entries)); } return sb.toString(); } diff -r 3527a32a19f0 -r 04a79821e760 callgraph/src/test/java/org/netbeans/lib/callgraph/ArgumentsTest.java --- a/callgraph/src/test/java/org/netbeans/lib/callgraph/ArgumentsTest.java Mon Aug 29 13:00:53 2016 -0400 +++ b/callgraph/src/test/java/org/netbeans/lib/callgraph/ArgumentsTest.java Sat Sep 03 02:41:36 2016 -0400 @@ -175,4 +175,59 @@ } } + + @Test + public void testAntScan() throws IOException { + File tmp = new File(System.getProperty("java.io.tmpdir")); + File dir = new File(tmp, ArgumentsTest.class.getSimpleName() + "_" + System.currentTimeMillis()); + File project1 = new File(dir, "prj"); + File project2 = new File(dir, "prj2"); + File pom1 = new File(project1, "build.xml"); + File pom2 = new File(project2, "build.xml"); + + File java1 = new File(project1, "src"); + File java2 = new File(project2, "src"); + + assertTrue(java1.mkdirs()); + assertTrue(java2.mkdirs()); + pom1.createNewFile(); + pom2.createNewFile(); + dir = dir.getCanonicalFile(); + java1 = java1.getCanonicalFile(); + java2 = java2.getCanonicalFile(); + + Arguments a = new Arguments("-v", dir.getAbsolutePath()); + assertEquals(1, a.folders().size()); + assertEquals(dir, a.folders().iterator().next()); + List errors = new ArrayList<>(); + a.findAntSubfolders(errors); + assertTrue(errors.isEmpty()); + System.out.println("FLDS: "); + for (File f : a.folders()) { + System.out.println(" " + f.getAbsolutePath()); + } + assertEquals(2, a.folders().size()); + assertFalse(a.folders().isEmpty()); + assertTrue(a.folders().contains(java1)); + assertTrue(a.folders().contains(java2)); + try { + + Arguments args = new Arguments("--ant", "-s", dir.getAbsolutePath()); + assertTrue(args.isAnt()); + assertTrue(args.isShortNames()); + assertTrue(args.isSelfReferences()); + assertTrue(args.folders().contains(java1)); + assertTrue(args.folders().contains(java2)); + } finally { + // Clean up + assertTrue(pom1.delete()); + assertTrue(pom2.delete()); + assertTrue(java1.delete()); + assertTrue(java2.delete()); + assertTrue(project1.delete()); + assertTrue(project2.delete()); + assertTrue(dir.delete()); + } + } + } diff -r 3527a32a19f0 -r 04a79821e760 callgraph/src/test/java/org/netbeans/lib/callgraph/util/SmallStringTest.java --- a/callgraph/src/test/java/org/netbeans/lib/callgraph/util/SmallStringTest.java Mon Aug 29 13:00:53 2016 -0400 +++ b/callgraph/src/test/java/org/netbeans/lib/callgraph/util/SmallStringTest.java Sat Sep 03 02:41:36 2016 -0400 @@ -9,6 +9,7 @@ import java.util.Set; import org.junit.Test; import static org.junit.Assert.*; +import org.netbeans.lib.callgraph.util.EightBitStrings.Concatenation; /** * @@ -120,7 +121,51 @@ Collections.sort(l); assertEquals(l, Arrays.asList(a, b, c, d, e)); } + + List stringsOf(List cs) { + List result = new ArrayList<>(); + for (CharSequence c : cs) { + if (c == null) { + break; + } + result.add(c.toString()); + } + return result; + } + + @Test + public void testAggressive() { + EightBitStrings strings = new EightBitStrings(false, true); + ComparableCharSequence c = strings.concat("com.foo.", "bar.baz.", "Hey$You"); + assertEquals("com.foo.bar.baz.Hey$You", c.toString()); + ComparableCharSequence c2 = strings.concat("com.foo.", "bar.whoodle.", "Hey$You"); + List interned = stringsOf(strings.dumpInternTable()); + for (String cs : interned) { + if (cs == null) { + break; + } + System.out.println(" " + cs); + } + assertTrue(interned.contains("com")); + assertTrue(interned.contains("foo")); + assertTrue(interned.contains("bar")); + assertTrue(interned.contains("baz")); + assertTrue(interned.contains("Hey")); + assertTrue(interned.contains("You")); + assertTrue(interned.contains("whoodle")); + assertTrue(interned.contains(".")); + assertTrue(interned.contains("$")); + assertTrue(interned.contains("\"")); + } + @Test + public void testConcatQuoted() { + EightBitStrings strings = new EightBitStrings(false); + List l = Arrays.asList(strings.create("hey"), false, 23, strings.create("bar"), "baz"); + CharSequence cc = strings.concatQuoted(l); + assertEquals("\"hey\" false 23 \"bar\" \"baz\"", cc.toString()); + } + private static String randomString(int len) { char[] c = new char[len]; for (int i = 0; i < c.length; i++) {