Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add clang qualtype to methods' parameter's type #58

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

feat: add clang qualtype to methods' parameter's type #58

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a545d0d
add clang qualtype to methods' parameter's type
Jan 17, 2025
8ddb74a
update unit test to fix unexpected errors in new ubuntu environment
Jan 17, 2025
3d0faea
add clang_qualtype parse unit test
Jan 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
256 changes: 7 additions & 249 deletions cxx-parser/src/constructor_initializer_parser.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
import { strict as assert } from 'assert';

import { execSync } from 'child_process';
import * as fs from 'fs';
import path from 'path';
import {
TagUsedType,
ClangASTNodeKind,
_FlattenNode,
dumpClangASTJSON,
filterAndFlattenNodes,
} from './utils'

import { ParseResult, visibleForTesting } from '@agoraio-extensions/terra-core';

Expand All @@ -17,47 +19,6 @@ import {
Struct,
} from './cxx_terra_node';

enum TagUsedType {
struct_t = 'struct',
}

/**
* The node type:
* https://clang.llvm.org/doxygen/Decl_8h_source.html
*/
enum ClangASTNodeKind {
TranslationUnitDecl = 'TranslationUnitDecl',
NamespaceDecl = 'NamespaceDecl',
TypedefDecl = 'TypedefDecl',
CXXRecordDecl = 'CXXRecordDecl',
FieldDecl = 'FieldDecl',
FullComment = 'FullComment',
ParagraphComment = 'ParagraphComment',
TextComment = 'TextComment',
InlineCommandComment = 'InlineCommandComment',
CXXConstructorDecl = 'CXXConstructorDecl',
CXXCtorInitializer = 'CXXCtorInitializer',
CXXBoolLiteralExpr = 'CXXBoolLiteralExpr',
ParmVarDecl = 'ParmVarDecl',
ImplicitCastExpr = 'ImplicitCastExpr',
CXXConstructExpr = 'CXXConstructExpr',
GNUNullExpr = 'GNUNullExpr',
CXXNullPtrLiteralExpr = 'CXXNullPtrLiteralExpr',
IntegerLiteral = 'IntegerLiteral',
FloatingLiteral = 'FloatingLiteral',
DeclRefExpr = 'DeclRefExpr',
EnumConstantDecl = 'EnumConstantDecl',
UnaryOperator = 'UnaryOperator',
}

/**
* Intermediate objects used internally
*/
interface _FlattenNode {
ns: string; // namespace
node: any;
}

/**
* Intermediate objects used internally
*/
Expand Down Expand Up @@ -86,161 +47,6 @@ class _StructConstructors {
constructors: _ConstructorInitializer[] = [];
}

/**
*
* @returns The default include dirs for clang command line tool
*/
function getDefaultIncludeDirs(): string[] {
function _findIncludeDirsFromClang(): string[] {
let verbose_output: string = '';
let verboseCommand = 'clang++ -xc++ -v -';
try {
execSync(verboseCommand, {
input: '',
stdio: ['pipe', 'pipe', 'pipe'], // Pipe stdin, stdout, and stderr
encoding: 'utf-8',
}).toString();
} catch (error: any) {
// We need to use the tricky wayt to get the verbose output from clang
verbose_output = error.message ?? '';
}

// If the command failed, return the empty include header dirs
if (!verbose_output) {
return [];
}

let verboseOutputInLines = verbose_output.split('\n');

verboseOutputInLines = verboseOutputInLines.slice(
verboseOutputInLines.findIndex((it) => {
return it.startsWith('#include <...>');
}) + 1, // Do not include the index of '#include <...>'
verboseOutputInLines.findIndex((it) => {
return it.startsWith('End of search list.');
})
);
let includeDirs = verboseOutputInLines
.filter((it) => {
return it.startsWith(' ');
})
.map((it) => {
if (it.includes(' (')) {
// /Library/Developer/CommandLineTools/SDKs/MacOSX13.sdk/System/Library/Frameworks (framework directory)
return it.trim().split(' (')[0];
}

return it.trim();
});

return includeDirs.map((it) => path.resolve(it));
}

return [
..._findIncludeDirsFromClang(),
// Add cxx-parser/cxx/cppast_backend/include/system_fake
path.join(getCppAstBackendDir(), 'include', 'system_fake'),
];
}

/**
* This function mainly use the clang command line tool to dump the AST in json format,
* ```
* clang++ -fsyntax-only -Xclang -ast-dump=json \
* -I /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include \
* -I /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include/c++/v1/
* <parse-files> > output.json
* ```
* More detail: https://clang.llvm.org/docs/IntroductionToTheClangAST.html
*
* For debug, you can dump the normal AST format which is more readable,
* ```
* clang++ -fsyntax-only -Xclang -ast-dump -fno-color-diagnostics \
* -I /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include \
* -I /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include/c++/v1/ \
* <parse-files> > output.txt
* ````
*/
function dumpClangASTJSON(
buildDir: string,
includeHeaderDirs: string[],
parseFile: string
): string {
// In some systems, if the llvm path spefic to the `PATH` environment variables, the clang AST
// will run with error that can not find the std headers, e.g.,
// `/opt/homebrew/opt/llvm@15/include/c++/v1/stdio.h:107:15: fatal error: 'stdio.h' file not found`.
// So we filter the llvm path from the `PATH` to run the clang AST, it's a little tricky but for a workaround.
function _clangCLIEnv() {
let originalEnv = process.env;
let pathEnv = process.env.PATH;
if (pathEnv) {
pathEnv = pathEnv.split(':').filter((it) => !it.includes('llvm')).join(':');
}

return {
...originalEnv,
PATH: pathEnv,
}
}

let fileName = path.basename(parseFile);
let checksum = generateChecksum([parseFile]);

// dump_clang_ast_<file name>_<checksum>.json
let clangAstJsonPath = path.join(
buildDir,
`dump_clang_ast_${fileName}_${checksum}.json`
);

// If the cache found, just return it.
// Note that we are no need to handle the `TerraContext.clean` in this function, since if the
// clean flag passed from the command line, the `CXXParser` will handle it first, the build dir
// will be cleaned.
if (fs.existsSync(clangAstJsonPath)) {
return fs.readFileSync(clangAstJsonPath, 'utf-8');
}

let includeHeaderDirsNew = [...getDefaultIncludeDirs(), ...includeHeaderDirs];
let includeHeaderDirsArgs = includeHeaderDirsNew.map((it) => {
return `-I ${it}`;
});

let args = ['-Xclang', '-ast-dump=json', '-fsyntax-only', '-std=c++11'];
let _args = [...args, ...includeHeaderDirsArgs, parseFile].join(' ');

let bashScript = `clang++ ${_args} > ${clangAstJsonPath}`;
console.log(`Running command: \n${bashScript}`);

try {
execSync(bashScript, {
stdio: ['pipe', 'pipe', 'pipe'], // Pipe stdin, stdout, and stderr,
encoding: 'utf8',
env: _clangCLIEnv(),
});
} catch (err: any) {
let errMessage = err.message;
// Eliminate the fatal error summary, and then see if there's any other fatal error message, like
// /usr/local/Cellar/llvm@15/15.0.7/lib/clang/15.0.7/include/arm64intr.h:12:15: fatal error: 'arm64intr.h' file not found
errMessage = errMessage.replace(
'fatal error: too many errors emitted, stopping now [-ferror-limit=]',
''
);
if (errMessage.includes('fatal error:')) {
// The file in path `clangAstJsonPath` will be created no matter the clang command failed or not,
// so we need to remove it if the clang command failed.
if (fs.existsSync(clangAstJsonPath)) {
fs.rmSync(clangAstJsonPath);
}
console.error(err.message);
process.exit(err.status);
}

console.log(errMessage);
}

let ast_json_file_content = fs.readFileSync(clangAstJsonPath, 'utf-8');
return ast_json_file_content;
}

function _parseStructConstructors(
buildDir: string,
Expand Down Expand Up @@ -554,53 +360,5 @@ export function ClangASTStructConstructorParser(
});
}

function filterAndFlattenNodes(
parseFiles: string,
astJson: string
): Array<_FlattenNode> {
let jsonObj = JSON.parse(astJson);
// The first object kind is `TranslationUnitDecl`
assert(jsonObj.kind == ClangASTNodeKind.TranslationUnitDecl);
// `inner` should have values
assert(jsonObj.inner && Array.isArray(jsonObj.inner));

function _flattenNodes(
inner: any,
nsStack: string[],
filterByFile: boolean = true
): Array<_FlattenNode> {
let res = new Array<_FlattenNode>();

for (let n of inner) {
if (filterByFile) {
if (!n.loc?.file) {
continue;
}

if (!parseFiles.includes(n.loc.file)) {
continue;
}
}

switch (n.kind) {
case ClangASTNodeKind.NamespaceDecl:
nsStack.push(n.name);
res.push(..._flattenNodes(n.inner, nsStack, false));
nsStack.pop();
break;
default:
res.push({ ns: nsStack.join('::'), node: n });
break;
}
}

return res;
}

let res = _flattenNodes(jsonObj.inner, []);

return res;
}

// Add functions that visible for testing here.
visibleForTesting(module, parseStructConstructors);
12 changes: 12 additions & 0 deletions cxx-parser/src/cxx_parser.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import './cxx_parser_ext';
import { ParseResult, TerraContext } from '@agoraio-extensions/terra-core';

import { ClangASTStructConstructorParser } from './constructor_initializer_parser';
import { ClangASTQualTypeParser } from './qualtype_parser';
import { CXXParserConfigs, ParseFilesConfig } from './cxx_parser_configs';
import { CXXFile, CXXTYPE, CXXTerraNode, cast } from './cxx_terra_node';

Expand Down Expand Up @@ -154,6 +155,17 @@ export function CXXParser(
newParseResult
);

if(cxxParserConfigs.parseClangQualType){
ClangASTQualTypeParser(
getBuildDir(terraContext, cxxParserConfigs.buildDirNamePrefix),
cxxParserConfigs.includeHeaderDirs,
cppastParsedFiles,
newParseResult
);
}



return newParseResult;
}

Expand Down
2 changes: 2 additions & 0 deletions cxx-parser/src/cxx_parser_configs.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ export interface CXXParserConfigs {
includeHeaderDirs: string[];
definesMacros: string[];
parseFiles: ParseFilesConfig;
parseClangQualType:boolean;
}

export class CXXParserConfigs {
Expand All @@ -61,6 +62,7 @@ export class CXXParserConfigs {
})
.flat(1),
} as ParseFilesConfig,
parseClangQualType: original.parseClangQualType,
};
}
}
1 change: 1 addition & 0 deletions cxx-parser/src/cxx_terra_node.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,7 @@ export class SimpleType extends CXXTerraNode {
is_const: boolean = false;
is_builtin_type: boolean = false;
template_arguments: string[] = [];
clang_qualtype: string = '';

override get realName(): string {
if (this.name) {
Expand Down
Loading
Loading