[
{
"path": ".gitignore",
"content": "*.class\n*.lst\n.idea/*\n*.iml\n*.xml\n*.pyc\n\n"
},
{
"path": "CITATION.cff",
"content": "@inproceedings{\n alon2018codeseq,\n title={code2seq: Generating Sequences from Structured Representations of Code},\n author={Uri Alon and Shaked Brody and Omer Levy and Eran Yahav},\n booktitle={International Conference on Learning Representations},\n year={2019},\n url={https://openreview.net/forum?id=H1gKYo09tX},\n}\n"
},
{
"path": "CSharpExtractor/.gitattributes",
"content": "###############################################################################\n# Set default behavior to automatically normalize line endings.\n###############################################################################\n* text=auto\n\n###############################################################################\n# Set default behavior for command prompt diff.\n#\n# This is need for earlier builds of msysgit that does not have it on by\n# default for csharp files.\n# Note: This is only used by command line\n###############################################################################\n#*.cs diff=csharp\n\n###############################################################################\n# Set the merge driver for project and solution files\n#\n# Merging from the command prompt will add diff markers to the files if there\n# are conflicts (Merging from VS is not affected by the settings below, in VS\n# the diff markers are never inserted). Diff markers may cause the following \n# file extensions to fail to load in VS. An alternative would be to treat\n# these files as binary and thus will always conflict and require user\n# intervention with every merge. To do so, just uncomment the entries below\n###############################################################################\n#*.sln merge=binary\n#*.csproj merge=binary\n#*.vbproj merge=binary\n#*.vcxproj merge=binary\n#*.vcproj merge=binary\n#*.dbproj merge=binary\n#*.fsproj merge=binary\n#*.lsproj merge=binary\n#*.wixproj merge=binary\n#*.modelproj merge=binary\n#*.sqlproj merge=binary\n#*.wwaproj merge=binary\n\n###############################################################################\n# behavior for image files\n#\n# image files are treated as binary by default.\n###############################################################################\n#*.jpg binary\n#*.png binary\n#*.gif binary\n\n###############################################################################\n# diff behavior for common document formats\n# \n# Convert binary document formats to text before diffing them. This feature\n# is only available from the command line. Turn it on by uncommenting the \n# entries below.\n###############################################################################\n#*.doc diff=astextplain\n#*.DOC diff=astextplain\n#*.docx diff=astextplain\n#*.DOCX diff=astextplain\n#*.dot diff=astextplain\n#*.DOT diff=astextplain\n#*.pdf diff=astextplain\n#*.PDF diff=astextplain\n#*.rtf diff=astextplain\n#*.RTF diff=astextplain\n"
},
{
"path": "CSharpExtractor/.gitignore",
"content": "## Ignore Visual Studio temporary files, build results, and\n## files generated by popular Visual Studio add-ons.\n\n# User-specific files\n*.suo\n*.user\n*.userosscache\n*.sln.docstates\n\n# User-specific files (MonoDevelop/Xamarin Studio)\n*.userprefs\n\n# Build results\n[Dd]ebug/\n[Dd]ebugPublic/\n[Rr]elease/\n[Rr]eleases/\nx64/\nx86/\nbld/\n[Bb]in/\n[Oo]bj/\n[Ll]og/\n\n# Visual Studio 2015 cache/options directory\n.vs/\n# Uncomment if you have tasks that create the project's static files in wwwroot\n#wwwroot/\n\n# MSTest test Results\n[Tt]est[Rr]esult*/\n[Bb]uild[Ll]og.*\n\n# NUNIT\n*.VisualState.xml\nTestResult.xml\n\n# Build Results of an ATL Project\n[Dd]ebugPS/\n[Rr]eleasePS/\ndlldata.c\n\n# DNX\nproject.lock.json\nartifacts/\n\n*_i.c\n*_p.c\n*_i.h\n*.ilk\n*.meta\n*.obj\n*.pch\n*.pdb\n*.pgc\n*.pgd\n*.rsp\n*.sbr\n*.tlb\n*.tli\n*.tlh\n*.tmp\n*.tmp_proj\n*.log\n*.vspscc\n*.vssscc\n.builds\n*.pidb\n*.svclog\n*.scc\n\n# Chutzpah Test files\n_Chutzpah*\n\n# Visual C++ cache files\nipch/\n*.aps\n*.ncb\n*.opendb\n*.opensdf\n*.sdf\n*.cachefile\n*.VC.db\n*.VC.VC.opendb\n\n# Visual Studio profiler\n*.psess\n*.vsp\n*.vspx\n*.sap\n\n# TFS 2012 Local Workspace\n$tf/\n\n# Guidance Automation Toolkit\n*.gpState\n\n# ReSharper is a .NET coding add-in\n_ReSharper*/\n*.[Rr]e[Ss]harper\n*.DotSettings.user\n\n# JustCode is a .NET coding add-in\n.JustCode\n\n# TeamCity is a build add-in\n_TeamCity*\n\n# DotCover is a Code Coverage Tool\n*.dotCover\n\n# NCrunch\n_NCrunch_*\n.*crunch*.local.xml\nnCrunchTemp_*\n\n# MightyMoose\n*.mm.*\nAutoTest.Net/\n\n# Web workbench (sass)\n.sass-cache/\n\n# Installshield output folder\n[Ee]xpress/\n\n# DocProject is a documentation generator add-in\nDocProject/buildhelp/\nDocProject/Help/*.HxT\nDocProject/Help/*.HxC\nDocProject/Help/*.hhc\nDocProject/Help/*.hhk\nDocProject/Help/*.hhp\nDocProject/Help/Html2\nDocProject/Help/html\n\n# Click-Once directory\npublish/\n\n# Publish Web Output\n*.[Pp]ublish.xml\n*.azurePubxml\n# TODO: Comment the next line if you want to checkin your web deploy settings\n# but database connection strings (with potential passwords) will be unencrypted\n*.pubxml\n*.publishproj\n\n# Microsoft Azure Web App publish settings. Comment the next line if you want to\n# checkin your Azure Web App publish settings, but sensitive information contained\n# in these scripts will be unencrypted\nPublishScripts/\n\n# NuGet Packages\n*.nupkg\n# The packages folder can be ignored because of Package Restore\n**/packages/*\n# except build/, which is used as an MSBuild target.\n!**/packages/build/\n# Uncomment if necessary however generally it will be regenerated when needed\n#!**/packages/repositories.config\n# NuGet v3's project.json files produces more ignoreable files\n*.nuget.props\n*.nuget.targets\n\n# Microsoft Azure Build Output\ncsx/\n*.build.csdef\n\n# Microsoft Azure Emulator\necf/\nrcf/\n\n# Windows Store app package directories and files\nAppPackages/\nBundleArtifacts/\nPackage.StoreAssociation.xml\n_pkginfo.txt\n\n# Visual Studio cache files\n# files ending in .cache can be ignored\n*.[Cc]ache\n# but keep track of directories ending in .cache\n!*.[Cc]ache/\n\n# Others\nClientBin/\n~$*\n*~\n*.dbmdl\n*.dbproj.schemaview\n*.pfx\n*.publishsettings\nnode_modules/\norleans.codegen.cs\n\n# Since there are multiple workflows, uncomment next line to ignore bower_components\n# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)\n#bower_components/\n\n# RIA/Silverlight projects\nGenerated_Code/\n\n# Backup & report files from converting an old project file\n# to a newer Visual Studio version. Backup files are not needed,\n# because we have git ;-)\n_UpgradeReport_Files/\nBackup*/\nUpgradeLog*.XML\nUpgradeLog*.htm\n\n# SQL Server files\n*.mdf\n*.ldf\n\n# Business Intelligence projects\n*.rdl.data\n*.bim.layout\n*.bim_*.settings\n\n# Microsoft Fakes\nFakesAssemblies/\n\n# GhostDoc plugin setting file\n*.GhostDoc.xml\n\n# Node.js Tools for Visual Studio\n.ntvs_analysis.dat\n\n# Visual Studio 6 build log\n*.plg\n\n# Visual Studio 6 workspace options file\n*.opt\n\n# Visual Studio LightSwitch build output\n**/*.HTMLClient/GeneratedArtifacts\n**/*.DesktopClient/GeneratedArtifacts\n**/*.DesktopClient/ModelManifest.xml\n**/*.Server/GeneratedArtifacts\n**/*.Server/ModelManifest.xml\n_Pvt_Extensions\n\n# Paket dependency manager\n.paket/paket.exe\npaket-files/\n\n# FAKE - F# Make\n.fake/\n\n# JetBrains Rider\n.idea/\n*.sln.iml\n\n# no data\ndata/*\nbackupdata/*\n"
},
{
"path": "CSharpExtractor/CSharpExtractor/.nuget/packages.config",
"content": "\n\n \n \n \n \n \n \n \n"
},
{
"path": "CSharpExtractor/CSharpExtractor/CSharpExtractor.sln",
"content": "\nMicrosoft Visual Studio Solution File, Format Version 12.00\n# Visual Studio 15\nVisualStudioVersion = 15.0.28307.136\nMinimumVisualStudioVersion = 10.0.40219.1\nProject(\"{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}\") = \"Extractor\", \"Extractor\\Extractor.csproj\", \"{481EDE3F-0ED1-4CB9-814A-63A821022552}\"\nEndProject\nGlobal\n\tGlobalSection(SolutionConfigurationPlatforms) = preSolution\n\t\tDebug|Any CPU = Debug|Any CPU\n\t\tDebug|x64 = Debug|x64\n\t\tDebug|x86 = Debug|x86\n\t\tRelease|Any CPU = Release|Any CPU\n\t\tRelease|x64 = Release|x64\n\t\tRelease|x86 = Release|x86\n\t\tRelease20|Any CPU = Release20|Any CPU\n\t\tRelease20|x64 = Release20|x64\n\t\tRelease20|x86 = Release20|x86\n\tEndGlobalSection\n\tGlobalSection(ProjectConfigurationPlatforms) = postSolution\n\t\t{481EDE3F-0ED1-4CB9-814A-63A821022552}.Debug|Any CPU.ActiveCfg = Debug|Any CPU\n\t\t{481EDE3F-0ED1-4CB9-814A-63A821022552}.Debug|Any CPU.Build.0 = Debug|Any CPU\n\t\t{481EDE3F-0ED1-4CB9-814A-63A821022552}.Debug|x64.ActiveCfg = Debug|Any CPU\n\t\t{481EDE3F-0ED1-4CB9-814A-63A821022552}.Debug|x64.Build.0 = Debug|Any CPU\n\t\t{481EDE3F-0ED1-4CB9-814A-63A821022552}.Debug|x86.ActiveCfg = Debug|Any CPU\n\t\t{481EDE3F-0ED1-4CB9-814A-63A821022552}.Debug|x86.Build.0 = Debug|Any CPU\n\t\t{481EDE3F-0ED1-4CB9-814A-63A821022552}.Release|Any CPU.ActiveCfg = Release|Any CPU\n\t\t{481EDE3F-0ED1-4CB9-814A-63A821022552}.Release|Any CPU.Build.0 = Release|Any CPU\n\t\t{481EDE3F-0ED1-4CB9-814A-63A821022552}.Release|x64.ActiveCfg = Release|Any CPU\n\t\t{481EDE3F-0ED1-4CB9-814A-63A821022552}.Release|x64.Build.0 = Release|Any CPU\n\t\t{481EDE3F-0ED1-4CB9-814A-63A821022552}.Release|x86.ActiveCfg = Release|Any CPU\n\t\t{481EDE3F-0ED1-4CB9-814A-63A821022552}.Release|x86.Build.0 = Release|Any CPU\n\t\t{481EDE3F-0ED1-4CB9-814A-63A821022552}.Release20|Any CPU.ActiveCfg = Release|Any CPU\n\t\t{481EDE3F-0ED1-4CB9-814A-63A821022552}.Release20|Any CPU.Build.0 = Release|Any CPU\n\t\t{481EDE3F-0ED1-4CB9-814A-63A821022552}.Release20|x64.ActiveCfg = Release|Any CPU\n\t\t{481EDE3F-0ED1-4CB9-814A-63A821022552}.Release20|x64.Build.0 = Release|Any CPU\n\t\t{481EDE3F-0ED1-4CB9-814A-63A821022552}.Release20|x86.ActiveCfg = Release|Any CPU\n\t\t{481EDE3F-0ED1-4CB9-814A-63A821022552}.Release20|x86.Build.0 = Release|Any CPU\n\tEndGlobalSection\n\tGlobalSection(SolutionProperties) = preSolution\n\t\tHideSolutionNode = FALSE\n\tEndGlobalSection\n\tGlobalSection(ExtensibilityGlobals) = postSolution\n\t\tSolutionGuid = {13A0DA89-D5D9-4E75-850E-70B9FBE88FF8}\n\tEndGlobalSection\nEndGlobal\n"
},
{
"path": "CSharpExtractor/CSharpExtractor/Extractor/Extractor.cs",
"content": "using Extractor.Semantics;\nusing Microsoft.CodeAnalysis;\nusing Microsoft.CodeAnalysis.CSharp;\nusing Microsoft.CodeAnalysis.CSharp.Syntax;\nusing System;\nusing System.Collections.Generic;\nusing System.Linq;\nusing System.Text;\nusing System.Diagnostics;\n\n\nnamespace Extractor\n{\n public class Extractor\n {\n public const string InternalDelimiter = \"|\";\n public const string UpTreeChar = InternalDelimiter;\n public const string DownTreeChar = InternalDelimiter;\n public const string MethodNameConst = \"METHOD_NAME\";\n public static SyntaxKind[] ParentTypeToAddChildId = new SyntaxKind[] { SyntaxKind.SimpleAssignmentExpression,\n SyntaxKind.ElementAccessExpression, SyntaxKind.SimpleMemberAccessExpression, SyntaxKind.InvocationExpression, SyntaxKind.BracketedArgumentList, SyntaxKind.ArgumentList};\n\n private ICollection variables;\n\n public int LengthLimit { get; set; }\n public int WidthLimit { get; set; }\n public string Code { get; set; }\n public bool ShouldHash { get; set; }\n public int MaxContexts { get; set; }\n\n public Extractor(string code, Options opts)\n\t\t{\n LengthLimit = opts.MaxLength;\n WidthLimit = opts.MaxWidth;\n ShouldHash = !opts.NoHash;\n MaxContexts = opts.MaxContexts;\n Code = code;\n\t\t}\n\n\n\t\tStringBuilder builder = new StringBuilder();\n\n\t\tprivate string PathNodesToString(PathFinder.Path path)\n\t\t{\n\t\t\tbuilder.Clear();\n var nodeTypes = path.LeftSide;\n\t\t\tif (nodeTypes.Count() > 0)\n\t\t\t{\n\t\t\t\tbuilder.Append(nodeTypes.First().Kind());\n if (ParentTypeToAddChildId.Contains(nodeTypes.First().Parent.Kind()))\n {\n builder.Append(GetTruncatedChildId(nodeTypes.First()));\n }\n foreach (var n in nodeTypes.Skip(1))\n {\n builder.Append(UpTreeChar).Append(n.Kind());\n if (ParentTypeToAddChildId.Contains(n.Parent.Kind()))\n {\n builder.Append(GetTruncatedChildId(n));\n }\n }\n\t\t\t\tbuilder.Append(UpTreeChar);\n\t\t\t}\n\t\t\tbuilder.Append(path.Ancesstor.Kind());\n nodeTypes = path.RightSide;\n\t\t\tif (nodeTypes.Count() > 0)\n\t\t\t{\n\t\t\t\tbuilder.Append(DownTreeChar);\n\t\t\t\tbuilder.Append(nodeTypes.First().Kind());\n if (ParentTypeToAddChildId.Contains(nodeTypes.First().Parent.Kind()))\n {\n builder.Append(GetTruncatedChildId(nodeTypes.First()));\n }\n foreach (var n in nodeTypes.Skip(1))\n {\n builder.Append(DownTreeChar).Append(n.Kind());\n if (ParentTypeToAddChildId.Contains(n.Parent.Kind()))\n {\n builder.Append(GetTruncatedChildId(n));\n }\n }\n\t\t\t\t\n\t\t\t}\n\t\t\treturn builder.ToString();\n\t\t}\n\n private int GetTruncatedChildId(SyntaxNode n)\n {\n var parent = n.Parent;\n int index = parent.ChildNodes().ToList().IndexOf(n);\n if (index > 3)\n {\n index = 3;\n }\n return index;\n }\n\n private string PathToString(PathFinder.Path path)\n\t\t{\n\t\t\tSyntaxNode ancesstor = path.Ancesstor;\n\t\t\tStringBuilder builder = new StringBuilder();\n\t\t\tbuilder.Append(path.Left.Text).Append(UpTreeChar);\n\t\t\tbuilder.Append(this.PathNodesToString(path));\n\t\t\tbuilder.Append(DownTreeChar).Append(path.Right.Text);\n\t\t\treturn builder.ToString();\n\t\t}\n\n internal IEnumerable GetInternalPaths(Tree tree)\n {\n var finder = new PathFinder(tree, LengthLimit, WidthLimit);\n\n var allPairs = Utilities.ReservoirSample(Utilities.WeakConcat(Utilities.Choose2(variables),\n variables.Select((arg) => new Tuple(arg, arg))), MaxContexts);\n\n //iterate over variable-variable pairs\n foreach (Tuple varPair in allPairs)\n {\n bool pathToSelf = varPair.Item1 == varPair.Item2;\n\n foreach (var rhs in varPair.Item2.Leaves)\n foreach (var lhs in varPair.Item1.Leaves)\n \t\t\t\t{\n \n if (lhs == rhs)\n \t\t\t\t\t\tcontinue;\n\n PathFinder.Path path = finder.FindPath(lhs, rhs, limited: true);\n\n \t\t\t\t\tif (path == null)\n \t\t\t\t\t\tcontinue;\n \n yield return path;\n \t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t private string SplitNameUnlessEmpty(string original)\n\t {\n\t var subtokens = Utilities.SplitToSubtokens(original).Where(s => s.Length > 0);\n String name = String.Join(InternalDelimiter, subtokens);\n\t if (name.Length == 0)\n\t {\n\t name = Utilities.NormalizeName(original);\n\t }\n\n\t if (String.IsNullOrWhiteSpace(name))\n\t {\n\t name = \"SPACE\";\n\t }\n\n\t if (String.IsNullOrEmpty(name))\n\t {\n\t name = \"BLANK\";\n\t }\n if (original == Extractor.MethodNameConst)\n {\n name = original;\n }\n\t return name;\n }\n\n\n\t static readonly char[] removeFromComments = new char[] {' ', '/', '*', '{', '}'};\n\n public List Extract()\n\t\t{\n var tree = new Tree(CSharpSyntaxTree.ParseText(Code).GetRoot());\n\n IEnumerable methods = tree.GetRoot().DescendantNodesAndSelf().OfType().ToList();\n\n List results = new List();\n\n foreach(var method in methods) {\n\n String methodName = method.Identifier.ValueText;\n Tree methodTree = new Tree(method);\n var subtokensMethodName = Utilities.SplitToSubtokens(methodName);\n var tokenToVar = new Dictionary();\n this.variables = Variable.CreateFromMethod(methodTree).ToArray();\n\n foreach (var variable in variables)\n {\n foreach (SyntaxToken token in variable.Leaves)\n {\n tokenToVar[token] = variable;\n }\n }\n\n List contexts = new List();\n\n foreach (PathFinder.Path path in GetInternalPaths(methodTree))\n {\n String pathString = SplitNameUnlessEmpty(tokenToVar[path.Left].Name)\n + \",\" + MaybeHash(this.PathNodesToString(path))\n + \",\" + SplitNameUnlessEmpty(tokenToVar[path.Right].Name);\n\n Debug.WriteLine(path.Left.FullSpan+\" \"+tokenToVar[path.Left].Name+ \",\" +this.PathNodesToString(path)+ \",\" + tokenToVar[path.Right].Name+\" \"+path.Right.FullSpan); \n contexts.Add(pathString);\n }\n\n var commentNodes = tree.GetRoot().DescendantTrivia().Where(\n node => node.IsKind(SyntaxKind.MultiLineCommentTrivia) || node.IsKind(SyntaxKind.SingleLineCommentTrivia) || node.IsKind(SyntaxKind.MultiLineDocumentationCommentTrivia));\n foreach (SyntaxTrivia trivia in commentNodes)\n {\n\n string commentText = trivia.ToString().Trim(removeFromComments);\n\n string normalizedTrivia = SplitNameUnlessEmpty(commentText);\n var parts = normalizedTrivia.Split('|');\n for (int i = 0; i < Math.Ceiling((double)parts.Length / (double)5); i++)\n {\n var batch = String.Join(\"|\", parts.Skip(i * 5).Take(5));\n contexts.Add(batch + \",\" + \"COMMENT\" + \",\" + batch);\n }\n }\n results.Add(String.Join(\"|\", subtokensMethodName) + \" \" + String.Join(\" \", contexts)); \n }\n return results;\n }\n\n private string MaybeHash(string v)\n {\n if (this.ShouldHash)\n {\n return v.GetHashCode().ToString();\n } else\n {\n return v;\n }\n }\n }\n}\n"
},
{
"path": "CSharpExtractor/CSharpExtractor/Extractor/Extractor.csproj",
"content": "\n\n \n Exe\n netcoreapp2.2\n Extractor.Program\n \n\n \n \n \n\n \n \n \n\n \n \n \n \n\n\n"
},
{
"path": "CSharpExtractor/CSharpExtractor/Extractor/PathFinder.cs",
"content": "using Microsoft.CodeAnalysis;\nusing Microsoft.CodeAnalysis.CSharp.Syntax;\nusing System;\nusing System.Collections.Generic;\nusing System.Linq;\n\nnamespace Extractor\n{\n\n internal class PathFinder\n\t{\n\t\tinternal class Path\n\t\t{\n\t\t\tpublic SyntaxToken Left { get; }\n\t\t\tpublic List LeftSide { get; }\n\t\t\tpublic SyntaxNode Ancesstor { get; }\n\t\t\tpublic List RightSide { get; }\n\t\t\tpublic SyntaxToken Right { get; }\n\n\t\t\tpublic Path(SyntaxToken left, IEnumerable leftSide, SyntaxNode ancesstor, \n\t\t\t IEnumerable rightSide, SyntaxToken right)\n\t\t\t{\n\t\t\t\tthis.Left = left;\n\t\t\t\tthis.LeftSide = leftSide.ToList();\n\t\t\t\tthis.Ancesstor = ancesstor;\n\t\t\t\tthis.RightSide = rightSide.ToList();\n\t\t\t\tthis.Right = right;\n\t\t\t}\n\t\t}\n\n\t\tpublic int Length { get; }\n\t\tpublic int Width { get; }\n\n\t\tTree tree;\n\n\t\tpublic PathFinder(Tree tree, int length = 7, int width = 4)\n\t\t{\n\t\t\tif (length < 1 || width < 1)\n\t\t\t\tthrow new ArgumentException(\"Width and Length params must be positive.\");\n\n\t\t\tLength = length;\n\t\t\tWidth = width;\n\t\t\tthis.tree = tree;\n\t\t}\n\n\t\tprivate int GetDepth(SyntaxNode n)\n\t\t{\n int depth = 0;\n\t\t\twhile(n.Parent != null)\n {\n n = n.Parent;\n depth++;\n }\n return depth;\n\t\t}\n\n\t\tpublic SyntaxNode FirstAncestor(SyntaxNode l, SyntaxNode r)\n\t\t{\n\t\t\tif (l.Equals(r))\n\t\t\t\treturn l;\n\n\t\t\tif (GetDepth(l) >= GetDepth(r))\n\t\t\t{\n\t\t\t\tl = l.Parent;\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\tr = r.Parent;\n\t\t\t}\n\t\t\treturn FirstAncestor(l, r);\n\t\t}\n\n\t\tprivate IEnumerable CollectPathToParent(SyntaxNode start, SyntaxNode parent)\n\t\t{\n\t\t\twhile (!start.Equals(parent))\n\t\t\t{\n\t\t\t\tyield return start;\n\t\t\t\tstart = start.Parent;\n\t\t\t}\n\t\t}\n\n\t\tinternal Path FindPath(SyntaxToken l, SyntaxToken r, bool limited = true)\n\t\t{\n\t\t\tSyntaxNode p = FirstAncestor(l.Parent, r.Parent);\n\n\t\t\t// + 2 for the distance of the leafs themselves\n\t\t\tif (GetDepth(r.Parent) + GetDepth(l.Parent) - 2 * GetDepth(p) + 2 > Length)\n\t\t\t{\n\t\t\t\treturn null;\n\t\t\t}\n\n\t\t\tvar leftSide = CollectPathToParent(l.Parent, p);\n\t\t\tvar rightSide = CollectPathToParent(r.Parent, p);\n\t\t\trightSide = rightSide.Reverse();\n\n\t\t\tList widthCheck = p.ChildNodes().ToList();\n\t\t\tif (limited && leftSide.Count() != 0\n\t\t\t && rightSide.Count() != 0)\n\t\t\t{\n\t\t\t\tint indexOfLeft = widthCheck.IndexOf(leftSide.Last());\n\t\t\t\tint indexOfRight = widthCheck.IndexOf(rightSide.First());\n\t\t\t\tif (Math.Abs(indexOfLeft - indexOfRight) >= Width)\n\t\t\t\t{\n\t\t\t\t\treturn null;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn new Path(l, leftSide, p, rightSide, r);\n\t\t}\n\t}\n}\n"
},
{
"path": "CSharpExtractor/CSharpExtractor/Extractor/Program.cs",
"content": "using CommandLine;\nusing CommandLine.Text;\nusing System;\nusing System.Collections.Generic;\nusing System.IO;\nusing System.Linq;\n\nnamespace Extractor\n{\n class Program\n {\n static List ExtractSingleFile(string filename, Options opts)\n {\n string data = File.ReadAllText(filename);\n var extractor = new Extractor(data, opts);\n List result = extractor.Extract();\n\n return result;\n }\n\n static void Main(string[] args)\n {\n Options options = new Options();\n Parser.Default.ParseArguments(args)\n .WithParsed(opt => options = opt)\n .WithNotParsed(errors =>\n {\n Console.WriteLine(errors);\n return;\n });\n\n string path = options.Path;\n string[] files;\n if (Directory.Exists(path))\n {\n files = Directory.GetFiles(path, \"*.cs\", SearchOption.AllDirectories);\n }\n else\n {\n files = new string[] { path };\n }\n\n IEnumerable results = null;\n\n results = files.AsParallel().WithDegreeOfParallelism(options.Threads).SelectMany(filename => ExtractSingleFile(filename, options));\n\n using (StreamWriter sw = new StreamWriter(options.OFileName, append: true))\n {\n foreach (var res in results)\n {\n sw.WriteLine(res);\n }\n }\n }\n }\n}\n"
},
{
"path": "CSharpExtractor/CSharpExtractor/Extractor/Properties/launchSettings.json",
"content": "{\n \"profiles\": {\n \"Extractor\": {\n \"commandName\": \"Project\",\n \"commandLineArgs\": \"--path C:\\\\Users\\\\urial\\\\Source\\\\Repos\\\\CSharpExtractor\\\\CSharpExtractor\\\\Extractor\\\\bin\\\\ --no_hash\"\n }\n }\n}"
},
{
"path": "CSharpExtractor/CSharpExtractor/Extractor/Temp.cs",
"content": "namespace Extractor\n{\n class Temp\n {\n class NestedClass\n {\n void fooBar()\n {\n a.b = c;\n }\n }\n }\n}\n"
},
{
"path": "CSharpExtractor/CSharpExtractor/Extractor/Tree/Tree.cs",
"content": "using System;\nusing System.Collections.Generic;\nusing System.IO;\nusing System.Linq;\nusing System.Text;\nusing Microsoft.CodeAnalysis;\nusing Microsoft.CodeAnalysis.CSharp;\nusing Microsoft.CodeAnalysis.CSharp.Syntax;\n\nnamespace Extractor\n{\n public class Tree\n {\n public const string DummyClass = \"IgnoreDummyClass\";\n public const string DummyMethodName = \"IgnoreDummyMethod\";\n public const string DummyType = \"IgnoreDummyType\";\n internal static readonly SyntaxKind[] literals = { SyntaxKind.NumericLiteralToken, SyntaxKind.StringLiteralToken, SyntaxKind.CharacterLiteralToken };\n\n internal static readonly HashSet identifiers = new HashSet(new SyntaxKind[] { SyntaxKind.IdentifierToken }); //, SyntaxKind.VoidKeyword, SyntaxKind.StringKeyword });\n internal static readonly HashSet keywords = new HashSet(new SyntaxKind[] { SyntaxKind.RefKeyword, SyntaxKind.OutKeyword, SyntaxKind.ConstKeyword });\n internal static readonly HashSet declarations = new HashSet(new SyntaxKind[] { SyntaxKind.VariableDeclarator, SyntaxKind.Parameter, SyntaxKind.CatchDeclaration, SyntaxKind.ForEachStatement });\n internal static readonly HashSet memberAccesses = new HashSet(new SyntaxKind[] { SyntaxKind.SimpleMemberAccessExpression, SyntaxKind.PointerMemberAccessExpression });\n internal static readonly HashSet scopeEnders = new HashSet(\n new SyntaxKind[]{ SyntaxKind.Block, SyntaxKind.ForStatement, SyntaxKind.MethodDeclaration,\n SyntaxKind.ForEachStatement, SyntaxKind.CatchClause, SyntaxKind.SwitchSection, SyntaxKind.UsingStatement });\n\n internal static readonly HashSet lambdaScopeStarters = new HashSet(\n new SyntaxKind[]{ SyntaxKind.AnonymousMethodExpression,\n SyntaxKind.SimpleLambdaExpression, SyntaxKind.ParenthesizedLambdaExpression });\n\n public static bool IsScopeEnder(SyntaxNode node)\n {\n return Tree.scopeEnders.Contains(node.Kind());\n }\n\n class TreeBuilderWalker : CSharpSyntaxWalker\n {\n Dictionary nodes;\n HashSet visitedNodes;\n List Desc;\n List Tokens;\n Dictionary tokens;\n\n internal TreeBuilderWalker(Dictionary nodes, Dictionary tokens)\n {\n visitedNodes = new HashSet();\n this.nodes = nodes;\n this.tokens = tokens;\n }\n\n public override\n void Visit(SyntaxNode node)\n {\n visitedNodes.Add(node);\n\n base.Visit(node);\n\n visitedNodes.Remove(node);\n\n Desc = new List();\n Tokens = new List();\n foreach (var c in node.ChildNodes())\n {\n if (!nodes.ContainsKey(c))\n {\n continue;\n }\n Desc.AddRange(nodes[c].Descendents);\n Desc.Add(c);\n Tokens.AddRange(nodes[c].Leaves);\n }\n foreach (var token in node.ChildTokens())\n {\n if (Leaf.IsLeafToken(token))\n {\n tokens[token] = new Leaf(nodes, token);\n Tokens.Add(token);\n }\n }\n\n Node res = new Node(This: node,\n Ancestors: new HashSet(visitedNodes),\n Descendents: Desc.ToArray(),\n Leaves: Tokens.ToArray(),\n Kind: node.Kind());\n nodes[node] = res;\n\n }\n }\n\n internal SyntaxNode GetRoot()\n {\n return tree;\n }\n\n SyntaxNode tree;\n internal Dictionary nodes = new Dictionary();\n internal Dictionary leaves = new Dictionary();\n\n public Tree(SyntaxNode syntaxTree)\n {\n this.tree = syntaxTree;\n\n /*if (this.tree.ChildNodes().ToList().Count() == 0)\n {\n this.tree = CSharpSyntaxTree.ParseText($\"private {DummyType} {DummyMethodName}() {{ {code} }}\");\n }*/\n new TreeBuilderWalker(nodes, leaves).Visit(this.tree);\n\n List commentNodes = tree.DescendantTrivia().Where(\n node => node.IsKind(SyntaxKind.MultiLineCommentTrivia) || node.IsKind(SyntaxKind.SingleLineCommentTrivia)).ToList();\n\n }\n }\n\n public class Node\n {\n public Node(SyntaxNode This, HashSet Ancestors, SyntaxNode[] Descendents,\n SyntaxToken[] Leaves, SyntaxKind Kind)\n {\n this.This = This;\n this.Ancestors = Ancestors;\n this.Descendents = Descendents;\n this.AncestorsAndSelf = new HashSet(Ancestors);\n this.AncestorsAndSelf.Add(This);\n this.Leaves = Leaves;\n this.Depth = Depth;\n this.Kind = Kind;\n this.KindName = Kind.ToString();\n }\n\n public SyntaxNode This { get; }\n\n public HashSet Ancestors { get; }\n\n public HashSet AncestorsAndSelf { get; }\n\n public SyntaxNode[] Descendents { get; }\n\n public SyntaxToken[] Leaves { get; }\n\n public SyntaxKind Kind { get; }\n\n public string KindName { get; }\n\n public int Depth { get; }\n\n public override bool Equals(object obj)\n {\n var item = obj as Node;\n\n if (item == null)\n {\n return false;\n }\n\n return this.This.Equals(item.This);\n }\n\n public override int GetHashCode()\n {\n return this.This.GetHashCode();\n }\n }\n\n public class Leaf\n {\n internal static bool IsLeafToken(SyntaxToken token)\n {\n if (token.Text.Equals(\"var\") && token.IsKind(SyntaxKind.IdentifierToken)\n && token.Parent.IsKind(SyntaxKind.IdentifierName) && token.Parent.Parent.IsKind(SyntaxKind.VariableDeclaration)\n && token.Parent.Parent.Parent.IsKind(SyntaxKind.LocalDeclarationStatement))\n {\n return false;\n }\n\n if (token.ValueText == Tree.DummyMethodName || token.ValueText == Tree.DummyType)\n {\n return false;\n }\n\n return Tree.identifiers.Contains(token.Kind()) || Tree.literals.Contains(token.Kind()) || token.Parent.Kind() == SyntaxKind.PredefinedType;\n }\n\n public SyntaxToken token { get; }\n public SyntaxKind Kind { get; }\n public string KindName { get; }\n public string Text { get; set; }\n public bool IsConst { get; }\n public string VariableName { get; }\n\n public Leaf(Dictionary nodes, SyntaxToken token)\n {\n this.token = token;\n Kind = token.Kind();\n KindName = Kind.ToString();\n IsConst = !(Tree.identifiers.Contains(Kind) && Tree.declarations.Contains(token.Parent.Kind()));\n\n Text = token.ValueText;\n SyntaxNode node = token.Parent.Parent;\n SyntaxNode current = token.Parent;\n VariableName = Text;\n }\n }\n\n public class SyntaxViewer\n {\n private string ToDot(SyntaxTree tree)\n {\n List nodes = tree.GetRoot().DescendantNodesAndSelf().ToList();\n SyntaxToken[] tokens = tree.GetRoot().DescendantTokens().ToArray();\n\n string[] tokenStrings = tokens.Select((arg) => arg.Kind().ToString() + \"-\" + arg.ToString()).ToArray();\n string[] nodeStrings = nodes.Select((arg) => arg.Kind().ToString()).ToArray();\n\n Dictionary counts = new Dictionary();\n Dictionary nodeNames = new Dictionary();\n IEnumerable allItems = nodeStrings.Concat(tokenStrings);\n int i = 0;\n\n foreach (string name in allItems)\n {\n if (!counts.ContainsKey(name))\n counts[name] = 0;\n counts[name] += 1;\n\n nodeNames[i] = name + counts[name].ToString();\n i++;\n }\n\n StringBuilder builder = new StringBuilder();\n builder.AppendLine(\"digraph G {\");\n\n // vertexes\n for (i = 0; i < allItems.Count(); i++)\n {\n builder.AppendFormat(\"\\\"{0}\\\" ;\\n\", nodeNames[i]);\n }\n\n builder.AppendLine();\n\n // edges\n for (i = 1; i < nodes.Count(); i++)\n {\n builder.AppendFormat(\"\\\"{0}\\\"->\\\"{1}\\\" [];\\n\", nodeNames[nodes.IndexOf(nodes[i].Parent)], nodeNames[i]);\n }\n\n for (i = 0; i < tokens.Count(); i++)\n {\n builder.AppendFormat(\"\\\"{0}\\\"->\\\"{1}\\\" [];\\n\", nodeNames[nodes.IndexOf(tokens[i].Parent)], nodeNames[i + nodes.Count()]);\n }\n\n builder.AppendLine(\"}\");\n return builder.ToString();\n }\n\n public SyntaxViewer(SyntaxTree tree, string path = \"out.ong\")\n {\n\n string dotData = ToDot(tree);\n \n File.WriteAllText(\"out.dot\", dotData);\n }\n }\n}\n"
},
{
"path": "CSharpExtractor/CSharpExtractor/Extractor/Utilities.cs",
"content": "using CommandLine;\nusing System;\nusing System.Collections.Generic;\nusing System.Linq;\nusing System.Text;\nusing System.Diagnostics;\nusing System.Text.RegularExpressions;\n\nnamespace Extractor\n{\n public class Options\n {\n [Option('t', \"threads\", Default = 1, HelpText = \"How many threads to use <1>\")]\n public int Threads { get; set; }\n\n [Option('p', \"path\", Default = \"./data/\", HelpText = \"Where to find code files. <.>\")]\n public string Path { get; set; }\n\n [Option('l', \"max_length\", Default = 9, HelpText = \"Max path length\")]\n public int MaxLength { get; set; }\n\n [Option('l', \"max_width\", Default = 2, HelpText = \"Max path length\")]\n public int MaxWidth { get; set; }\n\n [Option('o', \"ofile_name\", Default = \"test.txt\", HelpText = \"Output file name\")]\n public String OFileName { get; set; }\n\n [Option('h', \"no_hash\", Default = true, HelpText = \"When enabled, prints the whole path strings (not hashed)\")]\n public Boolean NoHash { get; set; }\n\n [Option('l', \"max_contexts\", Default = 30000, HelpText = \"Max number of path contexts to sample. Affects only very large snippets\")]\n public int MaxContexts { get; set; }\n }\n\n public static class Utilities\n\t{\n\t public static String[] NumbericLiteralsToKeep = new String[] { \"0\", \"1\", \"2\", \"3\", \"4\", \"5\", \"10\" };\n public static IEnumerable> Choose2(IEnumerable enumerable)\n\t\t{\n\t\t\tint index = 0;\n\n\t\t\tforeach (var e in enumerable)\n\t\t\t{\n\t\t\t\t++index;\n\t\t\t\tforeach (var t in enumerable.Skip(index))\n\t\t\t\t\tyield return Tuple.Create(e, t);\n\t\t\t}\n\t\t}\n\n /// \n /// Sample uniform randomly numSamples from an enumerable, using reservoir sampling.\n /// See https://en.wikipedia.org/wiki/Reservoir_sampling\n /// \n /// \n /// \n /// \n /// \n public static IEnumerable ReservoirSample(this IEnumerable input, int numSamples)\n {\n var rng = new Random();\n var sampledElements = new List(numSamples);\n int seenElementCount = 0;\n foreach (var element in input)\n {\n seenElementCount++;\n if (sampledElements.Count < numSamples)\n {\n sampledElements.Add(element);\n }\n else\n {\n int position = rng.Next(seenElementCount);\n if (position < numSamples)\n {\n sampledElements[position] = element;\n }\n }\n }\n Debug.Assert(sampledElements.Count <= numSamples);\n return sampledElements;\n }\n\n\n public static IEnumerable WeakConcat(IEnumerable enumerable1, IEnumerable enumerable2)\n\t\t{\n\t\t\tforeach (T t in enumerable1)\n\t\t\t\tyield return t;\n\t\t\tforeach (T t in enumerable2)\n\t\t\t\tyield return t;\n\t\t}\n\n public static IEnumerable SplitToSubtokens(String name)\n {\n return Regex.Split(name.Trim(), \"(?<=[a-z])(?=[A-Z])|_|[0-9]|(?<=[A-Z])(?=[A-Z][a-z])|\\\\s+\")\n .Where(s => s.Length > 0)\n .Select(s => NormalizeName(s))\n .Where(s => s.Length > 0);\n }\n\n private static Regex Whitespaces = new Regex(@\"\\s\");\n private static Regex NonAlphabetic = new Regex(\"[^A-Za-z]\");\n\n public static String NormalizeName(string s)\n {\n String partiallyNormalized = s.ToLowerInvariant()\n .Replace(\"\\\\\\\\n\", String.Empty)\n .Replace(\"[\\\"',]\", String.Empty);\n\n partiallyNormalized = Whitespaces.Replace(partiallyNormalized, \"\");\n partiallyNormalized = Encoding.ASCII.GetString(\n Encoding.Convert(\n Encoding.UTF8,\n Encoding.GetEncoding(\n Encoding.ASCII.EncodingName,\n new EncoderReplacementFallback(string.Empty),\n new DecoderExceptionFallback()\n ),\n Encoding.UTF8.GetBytes(partiallyNormalized)\n )\n );\n\n if (partiallyNormalized.Contains('\\n'))\n {\n partiallyNormalized = partiallyNormalized.Replace('\\n', 'N');\n }\n if (partiallyNormalized.Contains('\\r'))\n {\n partiallyNormalized = partiallyNormalized.Replace('\\r', 'R');\n }\n if (partiallyNormalized.Contains(','))\n {\n partiallyNormalized = partiallyNormalized.Replace(',', 'C');\n }\n\n String completelyNormalized = NonAlphabetic.Replace(partiallyNormalized, String.Empty);\n if (completelyNormalized.Length == 0)\n {\n if (Regex.IsMatch(partiallyNormalized, @\"^\\d+$\"))\n {\n if (NumbericLiteralsToKeep.Contains(partiallyNormalized))\n {\n return partiallyNormalized;\n }\n else\n {\n return \"NUM\";\n }\n }\n\n return String.Empty;\n }\n return completelyNormalized;\n \n }\n }\n}\n"
},
{
"path": "CSharpExtractor/CSharpExtractor/Extractor/Variable.cs",
"content": "using System;\nusing System.Collections.Generic;\nusing System.Linq;\nusing Microsoft.CodeAnalysis;\nusing Microsoft.CodeAnalysis.CSharp.Syntax;\n\nnamespace Extractor\n{\n\tnamespace Semantics\n\t{\n\t\tpublic class Variable\n\t\t{\n\t\t\tTree tree;\n\n\t\t\tpublic string Name { get; }\n\t\t\tprivate HashSet leaves;\n\t\t\tpublic HashSet Leaves\n\t\t\t{\n\t\t\t\tget\n\t\t\t\t{\n\t\t\t\t\treturn leaves;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tprivate Nullable constant;\n\t\t\tpublic bool Const\n\t\t\t{\n\t\t\t\tget\n\t\t\t\t{\n\t\t\t\t\treturn constant.Value;\n\t\t\t\t}\n\t\t\t}\n\n\n\t\t\tprivate Variable(string name, SyntaxToken[] leaves, Tree tree)\n\t\t\t{\n\t\t\t\tthis.tree = tree;\n\t\t\t\tthis.Name = name;\n\t\t\t\tthis.leaves = new HashSet(leaves);\n\n\n\t\t\t\tconstant = true;\n\t\t\t\tforeach (var leaf in leaves)\n\t\t\t\t{\n\t\t\t\t\tif (!tree.leaves[leaf].IsConst)\n\t\t\t\t\t{\n\t\t\t\t\t\tconstant = false;\n\t\t\t\t\t\t// If not constant the it is a decleration token\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tpublic override int GetHashCode()\n\t\t\t{\n\t\t\t\treturn this.Name.GetHashCode();\n\t\t\t}\n\n\t\t\tpublic bool IsLiteral()\n\t\t\t{\n\t\t\t\treturn Tree.literals.Contains(tree.leaves[Leaves.First()].Kind);\n\t\t\t}\n\n internal static Boolean isMethodName(SyntaxToken token)\n {\n return token.Parent.IsKind(Microsoft.CodeAnalysis.CSharp.SyntaxKind.MethodDeclaration) \n && token.IsKind(Microsoft.CodeAnalysis.CSharp.SyntaxKind.IdentifierToken);\n }\n\n\t\t\t// Create a variable for each variable in scope from tokens while splitting identically named but differently scoped vars.\n\t\t\tinternal static IEnumerable CreateFromMethod(Tree methodTree)\n\t\t\t{\n\t\t\t var root = methodTree.nodes[methodTree.GetRoot()];\n\t\t\t\tvar leaves = root.Leaves.ToArray();\n\t\t\t\tDictionary tokenToName = new Dictionary();\n\t\t\t\tDictionary> nameToTokens = new Dictionary>();\n\t\t\t\tforeach (SyntaxToken token in root.Leaves)\n\t\t\t\t{\n\t\t\t\t\tstring name = methodTree.leaves[token].VariableName;\n if (isMethodName(token))\n {\n name = Extractor.MethodNameConst;\n }\n tokenToName[token] = name;\n\t\t\t\t\tif (!nameToTokens.ContainsKey(name))\n\t\t\t\t\t\tnameToTokens[name] = new List();\n\t\t\t\t\tnameToTokens[name].Add(token);\n\t\t\t\t}\n\n List results = new List();\n\n foreach (SyntaxToken leaf in leaves)\n\t\t\t\t{\n\t\t\t\t\tstring name = tokenToName[leaf];\n\t\t\t\t\tSyntaxToken[] syntaxTokens = nameToTokens[name].ToArray();\n var v = new Variable(name, syntaxTokens, methodTree);\n\n //check if exists\n var matches = results.Where(p => p.Name == name).ToList();\n bool alreadyExists = (matches.Count != 0);\n if (!alreadyExists)\n {\n results.Add(v);\n }\n }\n\n return results;\n\t\t\t}\n\t\t}\n\t}\n}\n"
},
{
"path": "CSharpExtractor/extract.py",
"content": "#!/usr/bin/python\n\nimport itertools\nimport multiprocessing\nimport os\nimport sys\nimport shutil\nimport subprocess\nfrom threading import Timer\nimport sys\nfrom argparse import ArgumentParser\nfrom subprocess import Popen, PIPE, STDOUT, call\n\n\n\ndef get_immediate_subdirectories(a_dir):\n return [(os.path.join(a_dir, name)) for name in os.listdir(a_dir)\n if os.path.isdir(os.path.join(a_dir, name))]\n\n\nTMP_DIR = \"\"\n\ndef ParallelExtractDir(args, dir):\n ExtractFeaturesForDir(args, dir, \"\")\n\n\ndef ExtractFeaturesForDir(args, dir, prefix):\n command = ['dotnet', 'run', '--project', args.csproj,\n '--max_length', str(args.max_path_length), '--max_width', str(args.max_path_width),\n '--path', dir, '--threads', str(args.num_threads), '--ofile_name', str(args.ofile_name)]\n\n\n # print command\n # os.system(command)\n kill = lambda process: process.kill()\n sleeper = subprocess.Popen(command, stderr=subprocess.PIPE)\n timer = Timer(600000, kill, [sleeper])\n\n try:\n timer.start()\n _, stderr = sleeper.communicate()\n finally:\n timer.cancel()\n\n if sleeper.poll() == 0:\n if len(stderr) > 0:\n print(sys.stderr, stderr)\n else:\n print(sys.stderr, 'dir: ' + str(dir) + ' was not completed in time')\n failed = True\n subdirs = get_immediate_subdirectories(dir)\n for subdir in subdirs:\n ExtractFeaturesForDir(args, subdir, prefix + dir.split('/')[-1] + '_')\n if failed:\n if os.path.exists(str(args.ofile_name)):\n os.remove(str(args.ofile_name))\n\ndef ExtractFeaturesForDirsList(args, dirs):\n global TMP_DIR\n TMP_DIR = \"./tmp/feature_extractor%d/\" % (os.getpid())\n if os.path.exists(TMP_DIR):\n shutil.rmtree(TMP_DIR, ignore_errors=True)\n os.makedirs(TMP_DIR)\n try:\n p = multiprocessing.Pool(4)\n p.starmap(ParallelExtractDir, zip(itertools.repeat(args), dirs))\n #for dir in dirs:\n # ExtractFeaturesForDir(args, dir, '')\n output_files = os.listdir(TMP_DIR)\n for f in output_files:\n os.system(\"cat %s/%s\" % (TMP_DIR, f))\n finally:\n shutil.rmtree(TMP_DIR, ignore_errors=True)\n\n\nif __name__ == '__main__':\n\n parser = ArgumentParser()\n parser.add_argument(\"-maxlen\", \"--max_path_length\", dest=\"max_path_length\", required=False, default=8)\n parser.add_argument(\"-maxwidth\", \"--max_path_width\", dest=\"max_path_width\", required=False, default=2)\n parser.add_argument(\"-threads\", \"--num_threads\", dest=\"num_threads\", required=False, default=64)\n parser.add_argument(\"--csproj\", dest=\"csproj\", required=True)\n parser.add_argument(\"-dir\", \"--dir\", dest=\"dir\", required=False)\n parser.add_argument(\"-ofile_name\", \"--ofile_name\", dest=\"ofile_name\", required=True)\n args = parser.parse_args()\n\n if args.dir is not None:\n subdirs = get_immediate_subdirectories(args.dir)\n to_extract = subdirs\n if len(subdirs) == 0:\n to_extract = [args.dir.rstrip('/')]\n ExtractFeaturesForDirsList(args, to_extract)\n"
},
{
"path": "Input.java",
"content": "public String getName() {\n\t\treturn name;\n\t}"
},
{
"path": "JavaExtractor/JPredict/.classpath",
"content": "\n\n\t\n\t\t\n\t\t\t\n\t\t\t\n\t\t\n\t\n\t\n\t\t\n\t\t\t\n\t\t\n\t\n\t\n\t\t\n\t\t\t\n\t\t\n\t\n\t\n\t\n\t\t\n\t\t\t\n\t\t\t\n\t\t\n\t\n\t\n\n"
},
{
"path": "JavaExtractor/JPredict/.gitignore",
"content": "/target/"
},
{
"path": "JavaExtractor/JPredict/src/main/java/JavaExtractor/App.java",
"content": "package JavaExtractor;\n\nimport JavaExtractor.Common.CommandLineValues;\nimport org.kohsuke.args4j.CmdLineException;\n\nimport java.io.IOException;\nimport java.nio.file.Files;\nimport java.nio.file.Paths;\nimport java.util.LinkedList;\nimport java.util.List;\nimport java.util.concurrent.ExecutionException;\nimport java.util.concurrent.Executors;\nimport java.util.concurrent.Future;\nimport java.util.concurrent.ThreadPoolExecutor;\n\npublic class App {\n private static CommandLineValues s_CommandLineValues;\n\n public static void main(String[] args) {\n try {\n s_CommandLineValues = new CommandLineValues(args);\n } catch (CmdLineException e) {\n e.printStackTrace();\n return;\n }\n\n if (s_CommandLineValues.File != null) {\n ExtractFeaturesTask extractFeaturesTask = new ExtractFeaturesTask(s_CommandLineValues,\n s_CommandLineValues.File.toPath());\n extractFeaturesTask.processFile();\n } else if (s_CommandLineValues.Dir != null) {\n extractDir();\n }\n }\n\n private static void extractDir() {\n ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(s_CommandLineValues.NumThreads);\n LinkedList tasks = new LinkedList<>();\n try {\n Files.walk(Paths.get(s_CommandLineValues.Dir)).filter(Files::isRegularFile)\n .filter(p -> p.toString().toLowerCase().endsWith(\".java\")).forEach(f -> {\n ExtractFeaturesTask task = new ExtractFeaturesTask(s_CommandLineValues, f);\n tasks.add(task);\n });\n } catch (IOException e) {\n e.printStackTrace();\n return;\n }\n List> tasksResults = null;\n try {\n tasksResults = executor.invokeAll(tasks);\n } catch (InterruptedException e) {\n e.printStackTrace();\n } finally {\n executor.shutdown();\n }\n tasksResults.forEach(f -> {\n try {\n f.get();\n } catch (InterruptedException | ExecutionException e) {\n e.printStackTrace();\n }\n });\n }\n}\n"
},
{
"path": "JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/CommandLineValues.java",
"content": "package JavaExtractor.Common;\n\nimport org.kohsuke.args4j.CmdLineException;\nimport org.kohsuke.args4j.CmdLineParser;\nimport org.kohsuke.args4j.Option;\n\nimport java.io.File;\n\n/**\n * This class handles the programs arguments.\n */\npublic class CommandLineValues {\n @Option(name = \"--file\", required = false)\n public File File = null;\n\n @Option(name = \"--dir\", required = false, forbids = \"--file\")\n public String Dir = null;\n\n @Option(name = \"--max_path_length\", required = true)\n public int MaxPathLength;\n\n @Option(name = \"--max_path_width\", required = true)\n public int MaxPathWidth;\n\n @Option(name = \"--num_threads\", required = false)\n public int NumThreads = 64;\n\n @Option(name = \"--min_code_len\", required = false)\n public int MinCodeLength = 1;\n\n @Option(name = \"--max_code_len\", required = false)\n public int MaxCodeLength = -1;\n\n @Option(name = \"--max_file_len\", required = false)\n public int MaxFileLength = -1;\n\n @Option(name = \"--pretty_print\", required = false)\n public boolean PrettyPrint = false;\n\n @Option(name = \"--max_child_id\", required = false)\n public int MaxChildId = 3;\n\n @Option(name = \"--json_output\", required = false)\n public boolean JsonOutput = false;\n\n public CommandLineValues(String... args) throws CmdLineException {\n CmdLineParser parser = new CmdLineParser(this);\n try {\n parser.parseArgument(args);\n } catch (CmdLineException e) {\n System.err.println(e.getMessage());\n parser.printUsage(System.err);\n throw e;\n }\n }\n\n public CommandLineValues() {\n\n }\n}"
},
{
"path": "JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/Common.java",
"content": "package JavaExtractor.Common;\n\nimport JavaExtractor.FeaturesEntities.Property;\nimport com.github.javaparser.ast.Node;\nimport com.github.javaparser.ast.UserDataKey;\n\nimport java.util.ArrayList;\nimport java.util.stream.Collectors;\nimport java.util.stream.Stream;\n\npublic final class Common {\n public static final UserDataKey PropertyKey = new UserDataKey() {\n };\n public static final UserDataKey ChildId = new UserDataKey() {\n };\n public static final String EmptyString = \"\";\n\n public static final String MethodDeclaration = \"MethodDeclaration\";\n public static final String NameExpr = \"NameExpr\";\n public static final String BlankWord = \"BLANK\";\n\n public static final int c_MaxLabelLength = 50;\n public static final String methodName = \"METHOD_NAME\";\n public static final String internalSeparator = \"|\";\n\n public static String normalizeName(String original, String defaultString) {\n original = original.toLowerCase().replaceAll(\"\\\\\\\\n\", \"\") // escaped new\n // lines\n .replaceAll(\"//s+\", \"\") // whitespaces\n .replaceAll(\"[\\\"',]\", \"\") // quotes, apostrophies, commas\n .replaceAll(\"\\\\P{Print}\", \"\"); // unicode weird characters\n String stripped = original.replaceAll(\"[^A-Za-z]\", \"\");\n if (stripped.length() == 0) {\n String carefulStripped = original.replaceAll(\" \", \"_\");\n if (carefulStripped.length() == 0) {\n return defaultString;\n } else {\n return carefulStripped;\n }\n } else {\n return stripped;\n }\n }\n\n public static boolean isMethod(Node node, String type) {\n Property parentProperty = node.getParentNode().getUserData(Common.PropertyKey);\n if (parentProperty == null) {\n return false;\n }\n\n String parentType = parentProperty.getType();\n return Common.NameExpr.equals(type) && Common.MethodDeclaration.equals(parentType);\n }\n\n public static ArrayList splitToSubtokens(String str1) {\n String str2 = str1.replace(\"|\", \" \");\n String str3 = str2.trim();\n return Stream.of(str3.split(\"(?<=[a-z])(?=[A-Z])|_|[0-9]|(?<=[A-Z])(?=[A-Z][a-z])|\\\\s+\"))\n .filter(s -> s.length() > 0).map(s -> Common.normalizeName(s, Common.EmptyString))\n .filter(s -> s.length() > 0).collect(Collectors.toCollection(ArrayList::new));\n }\n}\n"
},
{
"path": "JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/MethodContent.java",
"content": "package JavaExtractor.Common;\n\nimport com.github.javaparser.ast.Node;\n\nimport java.util.ArrayList;\n\npublic class MethodContent {\n private final ArrayList leaves;\n private final String name;\n\n private final String content;\n\n public MethodContent(ArrayList leaves, String name, String content) {\n this.leaves = leaves;\n this.name = name;\n this.content = content;\n }\n\n public ArrayList getLeaves() {\n return leaves;\n }\n\n public String getName() {\n return name;\n }\n\n public String getContent() {\n return content;\n }\n}\n"
},
{
"path": "JavaExtractor/JPredict/src/main/java/JavaExtractor/ExtractFeaturesTask.java",
"content": "package JavaExtractor;\n\nimport JavaExtractor.Common.CommandLineValues;\nimport JavaExtractor.Common.Common;\nimport JavaExtractor.FeaturesEntities.ProgramFeatures;\nimport org.apache.commons.lang3.StringUtils;\n\nimport java.io.IOException;\nimport java.nio.charset.Charset;\nimport java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.util.ArrayList;\nimport java.util.List;\nimport java.util.concurrent.Callable;\nimport com.google.gson.Gson;\n\nclass ExtractFeaturesTask implements Callable {\n private final CommandLineValues commandLineValues;\n private final Path filePath;\n\n public ExtractFeaturesTask(CommandLineValues commandLineValues, Path path) {\n this.commandLineValues = commandLineValues;\n this.filePath = path;\n }\n\n @Override\n public Void call() {\n processFile();\n return null;\n }\n\n public void processFile() {\n ArrayList features;\n try {\n features = extractSingleFile();\n } catch (IOException e) {\n e.printStackTrace();\n return;\n }\n if (features == null) {\n return;\n }\n\n String toPrint = featuresToString(features);\n if (toPrint.length() > 0) {\n System.out.println(toPrint);\n }\n }\n\n private ArrayList extractSingleFile() throws IOException {\n String code;\n\n if (commandLineValues.MaxFileLength > 0 &&\n Files.lines(filePath, Charset.defaultCharset()).count() > commandLineValues.MaxFileLength) {\n return new ArrayList<>();\n }\n try {\n code = new String(Files.readAllBytes(filePath));\n } catch (IOException e) {\n e.printStackTrace();\n code = Common.EmptyString;\n }\n FeatureExtractor featureExtractor = new FeatureExtractor(commandLineValues, this.filePath);\n\n return featureExtractor.extractFeatures(code);\n }\n\n public String featuresToString(ArrayList features) {\n if (features == null || features.isEmpty()) {\n return Common.EmptyString;\n }\n\n List methodsOutputs = new ArrayList<>();\n\n for (ProgramFeatures singleMethodFeatures : features) {\n StringBuilder builder = new StringBuilder();\n\n String toPrint;\n if (commandLineValues.JsonOutput) {\n toPrint = new Gson().toJson(singleMethodFeatures);\n }\n else {\n toPrint = singleMethodFeatures.toString();\n }\n if (commandLineValues.PrettyPrint) {\n toPrint = toPrint.replace(\" \", \"\\n\\t\");\n }\n builder.append(toPrint);\n\n\n methodsOutputs.add(builder.toString());\n\n }\n return StringUtils.join(methodsOutputs, \"\\n\");\n }\n}\n"
},
{
"path": "JavaExtractor/JPredict/src/main/java/JavaExtractor/FeatureExtractor.java",
"content": "package JavaExtractor;\n\nimport JavaExtractor.Common.CommandLineValues;\nimport JavaExtractor.Common.Common;\nimport JavaExtractor.Common.MethodContent;\nimport JavaExtractor.FeaturesEntities.ProgramFeatures;\nimport JavaExtractor.FeaturesEntities.Property;\nimport JavaExtractor.Visitors.FunctionVisitor;\nimport com.github.javaparser.JavaParser;\nimport com.github.javaparser.ParseProblemException;\nimport com.github.javaparser.ast.CompilationUnit;\nimport com.github.javaparser.ast.Node;\n\nimport java.io.File;\nimport java.nio.file.Path;\nimport java.util.ArrayList;\nimport java.util.HashSet;\nimport java.util.Set;\nimport java.util.StringJoiner;\nimport java.util.stream.Collectors;\nimport java.util.stream.Stream;\n\n@SuppressWarnings(\"StringEquality\")\nclass FeatureExtractor {\n private final static String upSymbol = \"|\";\n private final static String downSymbol = \"|\";\n private static final Set s_ParentTypeToAddChildId = Stream\n .of(\"AssignExpr\", \"ArrayAccessExpr\", \"FieldAccessExpr\", \"MethodCallExpr\")\n .collect(Collectors.toCollection(HashSet::new));\n private final CommandLineValues m_CommandLineValues;\n private final Path filePath;\n\n public FeatureExtractor(CommandLineValues commandLineValues, Path filePath) {\n this.m_CommandLineValues = commandLineValues;\n this.filePath = filePath;\n }\n\n private static ArrayList getTreeStack(Node node) {\n ArrayList upStack = new ArrayList<>();\n Node current = node;\n while (current != null) {\n upStack.add(current);\n current = current.getParentNode();\n }\n return upStack;\n }\n\n public ArrayList extractFeatures(String code) {\n CompilationUnit m_CompilationUnit = parseFileWithRetries(code);\n FunctionVisitor functionVisitor = new FunctionVisitor(m_CommandLineValues);\n\n functionVisitor.visit(m_CompilationUnit, null);\n\n ArrayList methods = functionVisitor.getMethodContents();\n\n return generatePathFeatures(methods);\n }\n\n private CompilationUnit parseFileWithRetries(String code) {\n final String classPrefix = \"public class Test {\";\n final String classSuffix = \"}\";\n final String methodPrefix = \"SomeUnknownReturnType f() {\";\n final String methodSuffix = \"return noSuchReturnValue; }\";\n\n String content = code;\n CompilationUnit parsed;\n try {\n parsed = JavaParser.parse(content);\n } catch (ParseProblemException e1) {\n // Wrap with a class and method\n try {\n content = classPrefix + methodPrefix + code + methodSuffix + classSuffix;\n parsed = JavaParser.parse(content);\n } catch (ParseProblemException e2) {\n // Wrap with a class only\n content = classPrefix + code + classSuffix;\n parsed = JavaParser.parse(content);\n }\n }\n\n return parsed;\n }\n\n private ArrayList generatePathFeatures(ArrayList methods) {\n ArrayList methodsFeatures = new ArrayList<>();\n for (MethodContent content : methods) {\n ProgramFeatures singleMethodFeatures = generatePathFeaturesForFunction(content);\n if (!singleMethodFeatures.isEmpty()) {\n methodsFeatures.add(singleMethodFeatures);\n }\n }\n return methodsFeatures;\n }\n\n private ProgramFeatures generatePathFeaturesForFunction(MethodContent methodContent) {\n ArrayList functionLeaves = methodContent.getLeaves();\n ProgramFeatures programFeatures = new ProgramFeatures(\n methodContent.getName(), this.filePath, methodContent.getContent());\n\n for (int i = 0; i < functionLeaves.size(); i++) {\n for (int j = i + 1; j < functionLeaves.size(); j++) {\n String separator = Common.EmptyString;\n\n String path = generatePath(functionLeaves.get(i), functionLeaves.get(j), separator);\n if (path != Common.EmptyString) {\n Property source = functionLeaves.get(i).getUserData(Common.PropertyKey);\n Property target = functionLeaves.get(j).getUserData(Common.PropertyKey);\n programFeatures.addFeature(source, path, target);\n }\n }\n }\n return programFeatures;\n }\n\n private String generatePath(Node source, Node target, String separator) {\n\n StringJoiner stringBuilder = new StringJoiner(separator);\n ArrayList sourceStack = getTreeStack(source);\n ArrayList targetStack = getTreeStack(target);\n\n int commonPrefix = 0;\n int currentSourceAncestorIndex = sourceStack.size() - 1;\n int currentTargetAncestorIndex = targetStack.size() - 1;\n while (currentSourceAncestorIndex >= 0 && currentTargetAncestorIndex >= 0\n && sourceStack.get(currentSourceAncestorIndex) == targetStack.get(currentTargetAncestorIndex)) {\n commonPrefix++;\n currentSourceAncestorIndex--;\n currentTargetAncestorIndex--;\n }\n\n int pathLength = sourceStack.size() + targetStack.size() - 2 * commonPrefix;\n if (pathLength > m_CommandLineValues.MaxPathLength) {\n return Common.EmptyString;\n }\n\n if (currentSourceAncestorIndex >= 0 && currentTargetAncestorIndex >= 0) {\n int pathWidth = targetStack.get(currentTargetAncestorIndex).getUserData(Common.ChildId)\n - sourceStack.get(currentSourceAncestorIndex).getUserData(Common.ChildId);\n if (pathWidth > m_CommandLineValues.MaxPathWidth) {\n return Common.EmptyString;\n }\n }\n\n for (int i = 0; i < sourceStack.size() - commonPrefix; i++) {\n Node currentNode = sourceStack.get(i);\n String childId = Common.EmptyString;\n String parentRawType = currentNode.getParentNode().getUserData(Common.PropertyKey).getRawType();\n if (i == 0 || s_ParentTypeToAddChildId.contains(parentRawType)) {\n childId = saturateChildId(currentNode.getUserData(Common.ChildId))\n .toString();\n }\n stringBuilder.add(String.format(\"%s%s%s\",\n currentNode.getUserData(Common.PropertyKey).getType(true), childId, upSymbol));\n }\n\n Node commonNode = sourceStack.get(sourceStack.size() - commonPrefix);\n String commonNodeChildId = Common.EmptyString;\n Property parentNodeProperty = commonNode.getParentNode().getUserData(Common.PropertyKey);\n String commonNodeParentRawType = Common.EmptyString;\n if (parentNodeProperty != null) {\n commonNodeParentRawType = parentNodeProperty.getRawType();\n }\n if (s_ParentTypeToAddChildId.contains(commonNodeParentRawType)) {\n commonNodeChildId = saturateChildId(commonNode.getUserData(Common.ChildId))\n .toString();\n }\n stringBuilder.add(String.format(\"%s%s\",\n commonNode.getUserData(Common.PropertyKey).getType(true), commonNodeChildId));\n\n for (int i = targetStack.size() - commonPrefix - 1; i >= 0; i--) {\n Node currentNode = targetStack.get(i);\n String childId = Common.EmptyString;\n if (i == 0 || s_ParentTypeToAddChildId.contains(currentNode.getUserData(Common.PropertyKey).getRawType())) {\n childId = saturateChildId(currentNode.getUserData(Common.ChildId))\n .toString();\n }\n stringBuilder.add(String.format(\"%s%s%s\", downSymbol,\n currentNode.getUserData(Common.PropertyKey).getType(true), childId));\n }\n\n return stringBuilder.toString();\n }\n\n private Integer saturateChildId(int childId) {\n return Math.min(childId, m_CommandLineValues.MaxChildId);\n }\n}\n"
},
{
"path": "JavaExtractor/JPredict/src/main/java/JavaExtractor/FeaturesEntities/ProgramFeatures.java",
"content": "package JavaExtractor.FeaturesEntities;\n\nimport java.nio.file.Path;\nimport java.util.ArrayList;\nimport java.util.stream.Collectors;\n\npublic class ProgramFeatures {\n String name;\n\n transient ArrayList features = new ArrayList<>();\n String textContent;\n\n String filePath;\n\n public ProgramFeatures(String name, Path filePath, String textContent) {\n\n this.name = name;\n this.filePath = filePath.toAbsolutePath().toString();\n this.textContent = textContent;\n }\n\n @SuppressWarnings(\"StringBufferReplaceableByString\")\n @Override\n public String toString() {\n StringBuilder stringBuilder = new StringBuilder();\n stringBuilder.append(name).append(\" \");\n stringBuilder.append(features.stream().map(ProgramRelation::toString).collect(Collectors.joining(\" \")));\n\n return stringBuilder.toString();\n }\n\n public void addFeature(Property source, String path, Property target) {\n ProgramRelation newRelation = new ProgramRelation(source, target, path);\n features.add(newRelation);\n }\n\n public boolean isEmpty() {\n return features.isEmpty();\n }\n}\n"
},
{
"path": "JavaExtractor/JPredict/src/main/java/JavaExtractor/FeaturesEntities/ProgramRelation.java",
"content": "package JavaExtractor.FeaturesEntities;\n\npublic class ProgramRelation {\n Property source;\n Property target;\n String path;\n\n public ProgramRelation(Property sourceName, Property targetName, String path) {\n source = sourceName;\n target = targetName;\n this.path = path;\n }\n\n public String toString() {\n return String.format(\"%s,%s,%s\", source.getName(), path,\n target.getName());\n }\n}\n"
},
{
"path": "JavaExtractor/JPredict/src/main/java/JavaExtractor/FeaturesEntities/Property.java",
"content": "package JavaExtractor.FeaturesEntities;\n\nimport JavaExtractor.Common.Common;\nimport com.github.javaparser.ast.Node;\nimport com.github.javaparser.ast.expr.AssignExpr;\nimport com.github.javaparser.ast.expr.BinaryExpr;\nimport com.github.javaparser.ast.expr.IntegerLiteralExpr;\nimport com.github.javaparser.ast.expr.UnaryExpr;\nimport com.github.javaparser.ast.type.ClassOrInterfaceType;\n\nimport java.util.*;\nimport java.util.stream.Collectors;\nimport java.util.stream.Stream;\n\npublic class Property {\n public static final HashSet NumericalKeepValues = Stream.of(\"0\", \"1\", \"32\", \"64\")\n .collect(Collectors.toCollection(HashSet::new));\n private static final Map shortTypes = Collections.unmodifiableMap(new HashMap() {\n /**\n *\n */\n private static final long serialVersionUID = 1L;\n\n {\n put(\"ArrayAccessExpr\", \"ArAc\");\n put(\"ArrayBracketPair\", \"ArBr\");\n put(\"ArrayCreationExpr\", \"ArCr\");\n put(\"ArrayCreationLevel\", \"ArCrLvl\");\n put(\"ArrayInitializerExpr\", \"ArIn\");\n put(\"ArrayType\", \"ArTy\");\n put(\"AssertStmt\", \"Asrt\");\n put(\"AssignExpr:and\", \"AsAn\");\n put(\"AssignExpr:assign\", \"As\");\n put(\"AssignExpr:lShift\", \"AsLS\");\n put(\"AssignExpr:minus\", \"AsMi\");\n put(\"AssignExpr:or\", \"AsOr\");\n put(\"AssignExpr:plus\", \"AsP\");\n put(\"AssignExpr:rem\", \"AsRe\");\n put(\"AssignExpr:rSignedShift\", \"AsRSS\");\n put(\"AssignExpr:rUnsignedShift\", \"AsRUS\");\n put(\"AssignExpr:slash\", \"AsSl\");\n put(\"AssignExpr:star\", \"AsSt\");\n put(\"AssignExpr:xor\", \"AsX\");\n put(\"BinaryExpr:and\", \"And\");\n put(\"BinaryExpr:binAnd\", \"BinAnd\");\n put(\"BinaryExpr:binOr\", \"BinOr\");\n put(\"BinaryExpr:divide\", \"Div\");\n put(\"BinaryExpr:equals\", \"Eq\");\n put(\"BinaryExpr:greater\", \"Gt\");\n put(\"BinaryExpr:greaterEquals\", \"Geq\");\n put(\"BinaryExpr:less\", \"Ls\");\n put(\"BinaryExpr:lessEquals\", \"Leq\");\n put(\"BinaryExpr:lShift\", \"LS\");\n put(\"BinaryExpr:minus\", \"Minus\");\n put(\"BinaryExpr:notEquals\", \"Neq\");\n put(\"BinaryExpr:or\", \"Or\");\n put(\"BinaryExpr:plus\", \"Plus\");\n put(\"BinaryExpr:remainder\", \"Mod\");\n put(\"BinaryExpr:rSignedShift\", \"RSS\");\n put(\"BinaryExpr:rUnsignedShift\", \"RUS\");\n put(\"BinaryExpr:times\", \"Mul\");\n put(\"BinaryExpr:xor\", \"Xor\");\n put(\"BlockStmt\", \"Bk\");\n put(\"BooleanLiteralExpr\", \"BoolEx\");\n put(\"CastExpr\", \"Cast\");\n put(\"CatchClause\", \"Catch\");\n put(\"CharLiteralExpr\", \"CharEx\");\n put(\"ClassExpr\", \"ClsEx\");\n put(\"ClassOrInterfaceDeclaration\", \"ClsD\");\n put(\"ClassOrInterfaceType\", \"Cls\");\n put(\"ConditionalExpr\", \"Cond\");\n put(\"ConstructorDeclaration\", \"Ctor\");\n put(\"DoStmt\", \"Do\");\n put(\"DoubleLiteralExpr\", \"Dbl\");\n put(\"EmptyMemberDeclaration\", \"Emp\");\n put(\"EnclosedExpr\", \"Enc\");\n put(\"ExplicitConstructorInvocationStmt\", \"ExpCtor\");\n put(\"ExpressionStmt\", \"Ex\");\n put(\"FieldAccessExpr\", \"Fld\");\n put(\"FieldDeclaration\", \"FldDec\");\n put(\"ForeachStmt\", \"Foreach\");\n put(\"ForStmt\", \"For\");\n put(\"IfStmt\", \"If\");\n put(\"InitializerDeclaration\", \"Init\");\n put(\"InstanceOfExpr\", \"InstanceOf\");\n put(\"IntegerLiteralExpr\", \"IntEx\");\n put(\"IntegerLiteralMinValueExpr\", \"IntMinEx\");\n put(\"LabeledStmt\", \"Labeled\");\n put(\"LambdaExpr\", \"Lambda\");\n put(\"LongLiteralExpr\", \"LongEx\");\n put(\"MarkerAnnotationExpr\", \"MarkerExpr\");\n put(\"MemberValuePair\", \"Mvp\");\n put(\"MethodCallExpr\", \"Cal\");\n put(\"MethodDeclaration\", \"Mth\");\n put(\"MethodReferenceExpr\", \"MethRef\");\n put(\"NameExpr\", \"Nm\");\n put(\"NormalAnnotationExpr\", \"NormEx\");\n put(\"NullLiteralExpr\", \"Null\");\n put(\"ObjectCreationExpr\", \"ObjEx\");\n put(\"Parameter\", \"Prm\");\n put(\"PrimitiveType\", \"Prim\");\n put(\"QualifiedNameExpr\", \"Qua\");\n put(\"ReturnStmt\", \"Ret\");\n put(\"SingleMemberAnnotationExpr\", \"SMEx\");\n put(\"StringLiteralExpr\", \"StrEx\");\n put(\"SuperExpr\", \"SupEx\");\n put(\"SwitchEntryStmt\", \"SwiEnt\");\n put(\"SwitchStmt\", \"Switch\");\n put(\"SynchronizedStmt\", \"Sync\");\n put(\"ThisExpr\", \"This\");\n put(\"ThrowStmt\", \"Thro\");\n put(\"TryStmt\", \"Try\");\n put(\"TypeDeclarationStmt\", \"TypeDec\");\n put(\"TypeExpr\", \"Type\");\n put(\"TypeParameter\", \"TypePar\");\n put(\"UnaryExpr:inverse\", \"Inverse\");\n put(\"UnaryExpr:negative\", \"Neg\");\n put(\"UnaryExpr:not\", \"Not\");\n put(\"UnaryExpr:posDecrement\", \"PosDec\");\n put(\"UnaryExpr:posIncrement\", \"PosInc\");\n put(\"UnaryExpr:positive\", \"Pos\");\n put(\"UnaryExpr:preDecrement\", \"PreDec\");\n put(\"UnaryExpr:preIncrement\", \"PreInc\");\n put(\"UnionType\", \"Unio\");\n put(\"VariableDeclarationExpr\", \"VDE\");\n put(\"VariableDeclarator\", \"VD\");\n put(\"VariableDeclaratorId\", \"VDID\");\n put(\"VoidType\", \"Void\");\n put(\"WhileStmt\", \"While\");\n put(\"WildcardType\", \"Wild\");\n }\n });\n private final String RawType;\n private String Type;\n private String SplitName;\n\n public Property(Node node, boolean isLeaf, boolean isGenericParent) {\n Class nodeClass = node.getClass();\n RawType = Type = nodeClass.getSimpleName();\n if (node instanceof ClassOrInterfaceType && ((ClassOrInterfaceType) node).isBoxedType()) {\n Type = \"PrimitiveType\";\n }\n String operator = \"\";\n if (node instanceof BinaryExpr) {\n operator = ((BinaryExpr) node).getOperator().toString();\n } else if (node instanceof UnaryExpr) {\n operator = ((UnaryExpr) node).getOperator().toString();\n } else if (node instanceof AssignExpr) {\n operator = ((AssignExpr) node).getOperator().toString();\n }\n if (operator.length() > 0) {\n Type += \":\" + operator;\n }\n\n String nameToSplit = node.toString();\n if (isGenericParent) {\n nameToSplit = ((ClassOrInterfaceType) node).getName();\n if (isLeaf) {\n // if it is a generic parent which counts as a leaf, then when\n // it is participating in a path\n // as a parent, it should be GenericClass and not a simple\n // ClassOrInterfaceType.\n Type = \"GenericClass\";\n }\n }\n ArrayList splitNameParts = Common.splitToSubtokens(nameToSplit);\n SplitName = String.join(Common.internalSeparator, splitNameParts);\n\n String name = Common.normalizeName(node.toString(), Common.BlankWord);\n if (name.length() > Common.c_MaxLabelLength) {\n name = name.substring(0, Common.c_MaxLabelLength);\n } else if (node instanceof ClassOrInterfaceType && ((ClassOrInterfaceType) node).isBoxedType()) {\n name = ((ClassOrInterfaceType) node).toUnboxedType().toString();\n }\n\n if (Common.isMethod(node, Type)) {\n name = SplitName = Common.methodName;\n }\n\n if (SplitName.length() == 0) {\n SplitName = name;\n if (node instanceof IntegerLiteralExpr && !NumericalKeepValues.contains(SplitName)) {\n // This is a numeric literal, but not in our white list\n SplitName = \"\";\n }\n }\n }\n\n public String getRawType() {\n return RawType;\n }\n\n public String getType() {\n return Type;\n }\n\n public String getType(boolean shorten) {\n if (shorten) {\n return shortTypes.getOrDefault(Type, Type);\n } else {\n return Type;\n }\n }\n\n public String getName() {\n return SplitName;\n }\n}\n"
},
{
"path": "JavaExtractor/JPredict/src/main/java/JavaExtractor/Visitors/FunctionVisitor.java",
"content": "package JavaExtractor.Visitors;\n\nimport JavaExtractor.Common.CommandLineValues;\nimport JavaExtractor.Common.Common;\nimport JavaExtractor.Common.MethodContent;\nimport com.github.javaparser.ast.Node;\nimport com.github.javaparser.ast.body.MethodDeclaration;\nimport com.github.javaparser.ast.visitor.VoidVisitorAdapter;\n\nimport java.util.ArrayList;\nimport java.util.Arrays;\n\n@SuppressWarnings(\"StringEquality\")\npublic class FunctionVisitor extends VoidVisitorAdapter