From 677be0f172de5ceed3e042a358fcf620c04f71a6 Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Wed, 13 Nov 2024 13:54:56 -0500 Subject: [PATCH 1/4] Update poetry build rules to include codeanalyzer-*.jar and add a test case to verify it exists. Signed-off-by: Rahul Krishna --- Makefile | 9 +- pyproject.toml | 2 +- tests/test_core.py | 61 +-- tests/tree_sitter/__init__.py | 0 tests/tree_sitter/c/__init__.py | 0 tests/tree_sitter/c/test_c_tree_sitter.py | 261 ------------- tests/tree_sitter/go/__init__.py | 0 tests/tree_sitter/go/test_go_tree_sitter.py | 306 --------------- tests/tree_sitter/javascript/__init__.py | 0 .../javascript/test_javascript_tree_sitter.py | 356 ------------------ tests/tree_sitter/python/__init__.py | 0 .../python/test_python_tree_sitter.py | 177 --------- 12 files changed, 42 insertions(+), 1130 deletions(-) delete mode 100644 tests/tree_sitter/__init__.py delete mode 100644 tests/tree_sitter/c/__init__.py delete mode 100644 tests/tree_sitter/c/test_c_tree_sitter.py delete mode 100644 tests/tree_sitter/go/__init__.py delete mode 100644 tests/tree_sitter/go/test_go_tree_sitter.py delete mode 100644 tests/tree_sitter/javascript/__init__.py delete mode 100644 tests/tree_sitter/javascript/test_javascript_tree_sitter.py delete mode 100644 tests/tree_sitter/python/__init__.py delete mode 100644 tests/tree_sitter/python/test_python_tree_sitter.py diff --git a/Makefile b/Makefile index 78bb48a..6fa37e2 100644 --- a/Makefile +++ b/Makefile @@ -49,4 +49,11 @@ refresh: ## Refresh code analyzer .PHONY: build build: ## Builds a new Python wheel $(info Building artifacts...) - poetry build + + # Inject the latest Code Analyzer JAR + wget -q $(shell curl -s https://api.github.com/repos/IBM/codenet-minerva-code-analyzer/releases/latest | jq -r '.assets[] | .browser_download_url') + mkdir -p cldk/analysis/java/codeanalyzer/jar/ + mv codeanalyzer-*.jar cldk/analysis/java/codeanalyzer/jar/ + + # Build the package + poetry build \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index b5b22d3..346b2f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ ] include = [ "LICENSE", - "cldk/analysis/java/codeanalyzer/jar/codeanalyzer.jar" + "cldk/analysis/java/codeanalyzer/jar/*.jar" ] [tool.poetry.dependencies] diff --git a/tests/test_core.py b/tests/test_core.py index 57d6920..d8030e7 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,28 +1,33 @@ -# def test_specifiy_codeanalyzer_backend_manually(test_fixture): -# # Initialize the CLDK object with the project directory, language, and backend. -# ns = CLDK( -# project_dir=test_fixture[0], -# language="java", -# backend="codeanalyzer", -# backend_path=test_fixture[1], -# analysis_db="/tmp", -# sdg=True, -# use_graalvm_binary=False, -# eager=True, -# ) -# classes_dict = ns.preprocessing.get_all_classes() -# assert len(classes_dict) > 0 - - -# def test_get_all_calsses(test_fixture): -# # Initialize the CLDK object with the project directory, language, and analysis_backend. -# ns = CLDK( -# project_dir=test_fixture[0], -# language="java", -# analysis_backend="codeanalyzer", -# analysis_json_path="/tmp", -# sdg=True, -# use_graalvm_binary=False, -# ) -# classes_dict = ns.analysis.get_all_classes() -# assert len(classes_dict) == 148 +"""Test core functionalities of the build process.""" + +import glob +from pdb import set_trace +from pathlib import Path +import re +from subprocess import run +import zipfile + +import pytest + + +@pytest.fixture(scope="session") +def build_project(): + # Run the Poetry build command + result = run(["poetry", "build"], capture_output=True, text=True) + assert result.returncode == 0, f"Poetry build failed: {result.stderr}" + + # Find the .whl file in the dist directory + wheel_files = glob.glob("dist/*.whl") + assert wheel_files, "No .whl file found in the dist directory" + return wheel_files[0] + + +def test_codeanalyzer_jar_in_wheel(build_project): + wheel_path = build_project + jar_pattern = re.compile(r"cldk/analysis/java/codeanalyzer/jar/codeanalyzer-.*\.jar") + + # Open the .whl file as a zip archive and check for the jar file + with zipfile.ZipFile(wheel_path, "r") as wheel_zip: + jar_files = [file for file in wheel_zip.namelist() if jar_pattern.match(file)] + + assert jar_files, "codeanalyzer-*.jar file not found in the wheel package" diff --git a/tests/tree_sitter/__init__.py b/tests/tree_sitter/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/tree_sitter/c/__init__.py b/tests/tree_sitter/c/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/tree_sitter/c/test_c_tree_sitter.py b/tests/tree_sitter/c/test_c_tree_sitter.py deleted file mode 100644 index 3c82415..0000000 --- a/tests/tree_sitter/c/test_c_tree_sitter.py +++ /dev/null @@ -1,261 +0,0 @@ -# from unittest import TestCase - -# from cldk.analysis.c.treesitter import CSitter - - -# class TestCTreeSitter(TestCase): -# """ -# Tests for C TreeSitter nodule -# """ - -# def setUp(self): -# """Runs before each test case""" -# self.c_ts = CSitter() - -# def tearDown(self): -# """Runs after each test case""" - -# def test_get_imports(self): -# code = """ -# #include -# #include "zmalloc.h" -# #include zz_a -# #include zz_b(a, b) - -# int main() { -# return 0; -# } -# """ - -# imports = self.c_ts.get_imports(code=code) -# self.assertEqual(len(imports), 4) -# self.assertEqual(imports[0].value, "stdlib.h") -# self.assertTrue(imports[0].is_system) -# self.assertEqual(imports[1].value, "zmalloc.h") -# self.assertFalse(imports[1].is_system) -# self.assertEqual(imports[2].value, "zz_a") -# self.assertFalse(imports[2].is_system) -# self.assertEqual(imports[3].value, "zz_b(a, b)") -# self.assertFalse(imports[3].is_system) - -# def test_get_translation_unit_details(self): -# code = """ -# #include -# #include "zmalloc.h" -# #include zz_a -# #include zz_b(a, b) - -# int main() { -# return 0; -# } -# """ - -# details = self.c_ts.get_translation_unit_details(code=code) -# self.assertEqual(len(details.imports), 4) -# self.assertEqual(len(details.functions), 1) - -# def test_get_all_methods(self): -# code = """ -# #include -# #include "zmalloc.h" - -# typedef void my_callback(void *, size_t); - -# static foo bar(); -# static baz quux(...); - -# pno client_t_AddClient(client_t *self) { /* code */ } - -# void * do_stuff(int arg1) { -# return 5; -# } -# """ - -# all_functions = self.c_ts.get_all_functions(code=code) -# self.assertEqual(len(all_functions), 2) -# self.assertEqual(all_functions[0].signature, "pno client_t_AddClient(client_t *self)") -# self.assertEqual(all_functions[0].name, "client_t_AddClient") -# self.assertEqual(all_functions[0].comment, "") -# self.assertEqual(all_functions[0].output.type, "pno") -# self.assertFalse(all_functions[0].output.is_reference) -# self.assertEqual(len(all_functions[0].parameters), 1) -# self.assertEqual(all_functions[0].parameters[0].name, "self") -# self.assertEqual(all_functions[0].parameters[0].type, "client_t*") -# self.assertTrue(all_functions[0].parameters[0].is_reference) - -# self.assertEqual(all_functions[1].output.type, "void*") -# self.assertTrue(all_functions[1].output.is_reference) - -# def test_return_function_pointer(self): -# code = """ -# int (*functionFactory(int n))(int, int) { -# printf("Got parameter %d", n); -# int (*functionPtr)(int,int) = &addInt; -# return functionPtr; -# } -# """ - -# all_functions = self.c_ts.get_all_functions(code=code) -# self.assertEqual(len(all_functions), 1) -# func = all_functions[0] -# self.assertEqual(func.signature, "int (*functionFactory(int n))(int, int)") -# self.assertEqual(func.name, "functionFactory") -# self.assertEqual(func.comment, "") -# self.assertEqual(func.output.type, "function") -# self.assertTrue(func.output.is_reference) -# self.assertEqual(len(func.parameters), 1) -# self.assertEqual(func.parameters[0].name, "n") -# self.assertEqual(func.parameters[0].type, "int") -# self.assertFalse(func.parameters[0].is_reference) - -# def test_function_array_parameter(self): -# code = """ -# void testing(int a[3][5]){ -# } -# """ - -# all_functions = self.c_ts.get_all_functions(code=code) -# self.assertEqual(len(all_functions), 1) -# func = all_functions[0] -# self.assertEqual(func.signature, "void testing(int a[3][5])") -# self.assertEqual(func.name, "testing") -# self.assertEqual(func.output.type, "void") -# self.assertEqual(len(func.parameters), 1) -# self.assertEqual(func.parameters[0].name, "a") -# self.assertEqual(func.parameters[0].type, "int[3][5]") -# self.assertFalse(func.parameters[0].is_reference) - -# def test_old_style_function_definition(self): -# code = """ -# // K&R style -# // test -# int foo(bar, baz, qux) -# unsigned int bar, baz; -# char *qux; -# { -# return 0; -# } -# """ - -# all_functions = self.c_ts.get_all_functions(code=code) -# self.assertEqual(len(all_functions), 1) -# func = all_functions[0] -# self.assertEqual(func.signature, "int foo(bar, baz, qux)\nunsigned int bar, baz;\nchar *qux;") -# self.assertEqual(func.name, "foo") -# self.assertEqual(func.comment, "// K&R style\n// test") -# self.assertEqual(func.output.type, "int") -# self.assertEqual(len(func.parameters), 3) -# self.assertEqual(func.parameters[0].name, "bar") -# self.assertEqual(func.parameters[0].type, "unsigned int") -# self.assertFalse(func.parameters[0].is_reference) -# self.assertEqual(func.parameters[1].name, "baz") -# self.assertEqual(func.parameters[1].type, "unsigned int") -# self.assertFalse(func.parameters[1].is_reference) -# self.assertEqual(func.parameters[2].name, "qux") -# self.assertEqual(func.parameters[2].type, "char*") -# self.assertTrue(func.parameters[2].is_reference) - -# def test_function_pointer_type_return(self): -# code = """ -# string* getFullName(string *name) -# { -# cout << "First: "; -# cin >> name[0]; -# cout << "Middle: "; -# cin >> name[1]; -# cout << "Last: "; -# cin >> name[2]; -# return name; -# } - -# void** func3(char *the_string) -# { -# } -# """ - -# all_functions = self.c_ts.get_all_functions(code=code) -# self.assertEqual(len(all_functions), 2) -# self.assertEqual(all_functions[0].name, "getFullName") -# self.assertEqual(all_functions[0].output.type, "string*") -# self.assertTrue(all_functions[0].output.is_reference) -# self.assertEqual(all_functions[1].name, "func3") -# self.assertEqual(all_functions[1].output.type, "void**") -# self.assertTrue(all_functions[1].output.is_reference) - -# def test_function_pointer_parameter(self): -# code = """ -# void test(int (* my_int_f)(int)) { -# int a, b, d; -# } -# """ - -# all_functions = self.c_ts.get_all_functions(code=code) -# self.assertEqual(len(all_functions), 1) -# func = all_functions[0] -# self.assertEqual(func.name, "test") -# self.assertEqual(len(func.parameters), 1) -# self.assertEqual(func.parameters[0].name, "my_int_f") -# self.assertEqual(func.parameters[0].type, "function") -# self.assertTrue(func.parameters[0].is_reference) - -# def test_function_specifiers(self): -# code = """ -# unsigned int static inline *do_stuff() { -# return 5; -# } -# """ - -# all_functions = self.c_ts.get_all_functions(code=code) -# self.assertEqual(len(all_functions), 1) -# func = all_functions[0] -# self.assertEqual(func.name, "do_stuff") -# self.assertEqual(func.output.type, "unsigned int*") -# self.assertTrue(func.output.is_reference) -# self.assertListEqual(func.specifiers, ["static", "inline"]) - -# def test_parameter_type_qualifiers(self): -# code = """ -# void func(const unsigned int x) { -# } -# """ - -# all_functions = self.c_ts.get_all_functions(code=code) -# self.assertEqual(len(all_functions), 1) -# func = all_functions[0] -# self.assertEqual(func.name, "func") -# self.assertEqual(len(func.parameters), 1) -# self.assertEqual(func.parameters[0].type, "unsigned int") -# self.assertEqual(func.parameters[0].name, "x") -# self.assertListEqual(func.parameters[0].qualifiers, ["const"]) - -# def test_variadic_parameter(self): -# code = """ -# void func(int x, ...) { -# } -# """ - -# all_functions = self.c_ts.get_all_functions(code=code) -# self.assertEqual(len(all_functions), 1) -# func = all_functions[0] -# self.assertEqual(func.name, "func") -# self.assertEqual(len(func.parameters), 2) -# self.assertEqual(func.parameters[0].type, "int") -# self.assertEqual(func.parameters[0].name, "x") -# self.assertEqual(func.parameters[1].type, "variadic") -# self.assertEqual(func.parameters[1].name, "...") - -# def test_complex_pointer_expression_parameter(self): -# code = """ -# int *do_stuff(int *((*arg1)[3][4])) { -# return 5; -# } -# """ - -# all_functions = self.c_ts.get_all_functions(code=code) -# self.assertEqual(len(all_functions), 1) -# func = all_functions[0] -# self.assertEqual(func.name, "do_stuff") -# self.assertEqual(len(func.parameters), 1) -# # TODO: not sure about this type -# self.assertEqual(func.parameters[0].type, "int*((*)[3][4])") -# self.assertEqual(func.parameters[0].name, "arg1") diff --git a/tests/tree_sitter/go/__init__.py b/tests/tree_sitter/go/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/tree_sitter/go/test_go_tree_sitter.py b/tests/tree_sitter/go/test_go_tree_sitter.py deleted file mode 100644 index e451fec..0000000 --- a/tests/tree_sitter/go/test_go_tree_sitter.py +++ /dev/null @@ -1,306 +0,0 @@ -# from unittest import TestCase - -# from cldk.analysis.go.treesitter import GoSitter - - -# class TestGoTreeSitter(TestCase): -# """ -# Tests for Go TreeSitter module -# """ - -# def setUp(self): -# """Runs before each test case""" -# self.go_ts = GoSitter() - -# def tearDown(self): -# """Runs after each test case""" - -# def test_get_imports(self): -# code = """ -# package main - -# import "fmt" -# import "lib/math" -# import _ "lib/some" - -# import() -# import ("strings") -# import ( -# "net/http" -# . "some/dsl" -# _ "os" -# alias "some/package" -# ) -# """ - -# imports = self.go_ts.get_imports(code=code) -# self.assertEqual(len(imports), 8) -# self.assertEqual(imports[0].path, "fmt") -# self.assertIsNone(imports[0].name) -# self.assertEqual(imports[2].path, "lib/some") -# self.assertEqual(imports[2].name, "_") -# self.assertEqual(imports[4].path, "net/http") -# self.assertIsNone(imports[4].name) -# self.assertEqual(imports[5].path, "some/dsl") -# self.assertEqual(imports[5].name, ".") -# self.assertEqual(imports[7].path, "some/package") -# self.assertEqual(imports[7].name, "alias") - -# def test_get_source_file_details(self): -# code = """ -# package main - -# import ( -# "fmt" -# "github.com/test/go-simple/internal" -# m "lib/math" -# . "lib/math" -# ) - -# type rect struct { -# width, height int -# } - -# func (r *rect) area() int { -# return r.width * r.height -# } -# """ - -# details = self.go_ts.get_source_file_details(source_file=code) -# self.assertEqual(len(details.imports), 4) -# self.assertEqual(len(details.callables), 1) - -# def test_get_all_callables(self): -# code = """ -# package main - -# type rect struct { -# width, height int -# } - -# func (r *rect) area() int { -# return r.width * r.height -# } - -# func main() { -# } -# """ - -# callables = self.go_ts.get_all_functions(code=code) -# self.assertEqual(len(callables), 2) -# self.assertEqual(callables[0].name, "area") -# self.assertEqual(callables[1].name, "main") - -# def test_get_method(self): -# code = """ -# package main - -# type rect struct { -# width, height int -# } - -# func (r *rect) area() int { -# return r.width * r.height -# } -# """ - -# callables = self.go_ts.get_all_functions(code=code) -# self.assertEqual(len(callables), 1) -# method = callables[0] -# self.assertEqual(method.name, "area") -# self.assertEqual(method.signature, "(r *rect) area() int") -# self.assertListEqual(method.modifiers, ["private"]) -# self.assertEqual(method.comment, "") -# self.assertEqual(method.code, "func (r *rect) area() int {\n return r.width * r.height\n }") -# self.assertEqual(method.start_line, 7) -# self.assertEqual(method.end_line, 9) -# self.assertListEqual(method.return_types, ["int"]) -# self.assertEqual(len(method.parameters), 0) -# self.assertIsNotNone(method.receiver) -# self.assertEqual(method.receiver.name, "r") -# self.assertEqual(method.receiver.type, "*rect") -# self.assertTrue(method.receiver.is_reference) -# self.assertFalse(method.receiver.is_variadic) - -# def test_get_callable_with_comment(self): -# code = """ -# package main - -# type rect struct { -# width, height int -# } - -# // area docs -# func (r *rect) area() int { -# return r.width * r.height -# } - -# // main docs -# // test -# func main() { -# } -# """ - -# callables = self.go_ts.get_all_functions(code=code) -# self.assertEqual(len(callables), 2) -# self.assertEqual(callables[0].name, "area") -# self.assertEqual(callables[0].comment, "// area docs") -# self.assertEqual(callables[1].name, "main") -# self.assertEqual(callables[1].comment, "// main docs\n// test") - -# def test_get_callable_with_multiple_returns(self): -# code = """ -# package main - -# func test(a int, b int, c float64) (float64, *[]int) { -# } - -# func test2() (a, b float64, func(p *T)) { -# } -# """ - -# callables = self.go_ts.get_all_functions(code=code) -# self.assertEqual(len(callables), 2) -# self.assertEqual(callables[0].name, "test") -# self.assertListEqual(callables[0].return_types, ["float64", "*[]int"]) -# self.assertEqual(callables[1].name, "test2") -# self.assertListEqual(callables[1].return_types, ["float64", "float64", "func(p *T)"]) - -# def test_get_generic_function(self): -# code = """ -# package main - -# // SumIntsOrFloats sums the values of map m. It supports both floats and integers -# // as map values. -# func SumIntsOrFloats[K comparable, V int64 | float64](m map[K]V) V { -# var s V -# for _, v := range m { -# s += v -# } -# return s -# } -# """ - -# callables = self.go_ts.get_all_functions(code=code) -# self.assertEqual(len(callables), 1) -# function = callables[0] -# self.assertEqual(function.name, "SumIntsOrFloats") -# self.assertEqual(function.signature, "SumIntsOrFloats[K comparable, V int64 | float64](m map[K]V) V") -# self.assertListEqual(function.modifiers, ["public"]) -# self.assertListEqual(function.return_types, ["V"]) -# self.assertEqual(function.comment, "// SumIntsOrFloats sums the values of map m. It supports both floats and integers\n// as map values.") -# self.assertEqual(len(function.parameters), 1) -# self.assertEqual(function.parameters[0].name, "m") -# self.assertEqual(function.parameters[0].type, "map[K]V") - -# def test_get_callable_with_variadic(self): -# code = """ -# package main - -# func test(a, b int, z float64, opt ...interface{}) (success bool) { -# } -# """ - -# callables = self.go_ts.get_all_functions(code=code) -# self.assertEqual(len(callables), 1) -# function = callables[0] -# self.assertEqual(function.name, "test") -# self.assertListEqual(function.modifiers, ["private"]) -# self.assertListEqual(function.return_types, ["bool"]) -# self.assertEqual(len(function.parameters), 4) -# self.assertEqual(function.parameters[0].name, "a") -# self.assertEqual(function.parameters[0].type, "int") -# self.assertEqual(function.parameters[1].name, "b") -# self.assertEqual(function.parameters[1].type, "int") -# self.assertEqual(function.parameters[2].name, "z") -# self.assertEqual(function.parameters[2].type, "float64") -# self.assertEqual(function.parameters[3].name, "opt") -# self.assertEqual(function.parameters[3].type, "...interface{}") -# self.assertTrue(function.parameters[3].is_variadic) - -# def test_get_callable_with_qualified_type(self): -# code = """ -# package main - -# import alias "some/package" - -# func test(a alias.SomeType) (bool) { -# } -# """ - -# callables = self.go_ts.get_all_functions(code=code) -# self.assertEqual(len(callables), 1) -# function = callables[0] -# self.assertEqual(function.name, "test") -# self.assertListEqual(function.modifiers, ["private"]) -# self.assertListEqual(function.return_types, ["bool"]) -# self.assertEqual(len(function.parameters), 1) -# self.assertEqual(function.parameters[0].name, "a") -# self.assertEqual(function.parameters[0].type, "alias.SomeType") - -# def test_get_callable_with_parms_without_name(self): -# code = """ -# package main - -# func test(int, ...int) (...int) { -# } -# """ - -# callables = self.go_ts.get_all_functions(code=code) -# self.assertEqual(len(callables), 1) -# function = callables[0] -# self.assertEqual(function.name, "test") -# self.assertListEqual(function.modifiers, ["private"]) -# self.assertListEqual(function.return_types, ["...int"]) -# self.assertEqual(len(function.parameters), 2) -# self.assertIsNone(function.parameters[0].name) -# self.assertEqual(function.parameters[0].type, "int") -# self.assertIsNone(function.parameters[1].name) -# self.assertEqual(function.parameters[1].type, "...int") -# self.assertTrue(function.parameters[1].is_variadic) - -# def test_get_callable_without_parms(self): -# code = """ -# package main - -# func test() (...int) { -# } -# """ - -# callables = self.go_ts.get_all_functions(code=code) -# self.assertEqual(len(callables), 1) -# function = callables[0] -# self.assertEqual(function.name, "test") -# self.assertListEqual(function.modifiers, ["private"]) -# self.assertListEqual(function.return_types, ["...int"]) -# self.assertEqual(len(function.parameters), 0) - -# def test_get_callable_without_return(self): -# code = """ -# package main - -# func test() { -# } -# """ - -# callables = self.go_ts.get_all_functions(code=code) -# self.assertEqual(len(callables), 1) -# function = callables[0] -# self.assertEqual(function.name, "test") -# self.assertListEqual(function.modifiers, ["private"]) -# self.assertListEqual(function.return_types, []) - -# def test_get_callable_with_typed_return(self): -# code = """ -# package main - -# func test() t[K, V] { -# } -# """ - -# callables = self.go_ts.get_all_functions(code=code) -# self.assertEqual(len(callables), 1) -# function = callables[0] -# self.assertEqual(function.name, "test") -# self.assertListEqual(function.modifiers, ["private"]) -# self.assertListEqual(function.return_types, ["t[K, V]"]) diff --git a/tests/tree_sitter/javascript/__init__.py b/tests/tree_sitter/javascript/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/tree_sitter/javascript/test_javascript_tree_sitter.py b/tests/tree_sitter/javascript/test_javascript_tree_sitter.py deleted file mode 100644 index 29aa2de..0000000 --- a/tests/tree_sitter/javascript/test_javascript_tree_sitter.py +++ /dev/null @@ -1,356 +0,0 @@ -# ################################################################################ -# # Copyright IBM Corporation 2024 -# # -# # Licensed under the Apache License, Version 2.0 (the "License"); -# # you may not use this file except in compliance with the License. -# # You may obtain a copy of the License at -# # -# # http://www.apache.org/licenses/LICENSE-2.0 -# # -# # Unless required by applicable law or agreed to in writing, software -# # distributed under the License is distributed on an "AS IS" BASIS, -# # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# # See the License for the specific language governing permissions and -# # limitations under the License. -# ################################################################################ - -# """ -# Javascript Treesitter Tests -# """ - -# from typing import List - -# from unittest import TestCase - -# from cldk.analysis.javascript.treesitter import JavascriptSitter -# from cldk.models.javascript.models import JsCallable - - -# class TestJavascriptTreeSitter(TestCase): -# """ -# Tests for Javascript TreeSitter module -# """ - -# def setUp(self): -# """Runs before each test case""" -# self.js_ts = JavascriptSitter() - -# def tearDown(self): -# """Runs after each test case""" - -# def test_get_source_file_details(self): -# code = """ -# import "module-name"; - -# const toCamelCase = str => { -# return str.toLowerCase().replace(/[-_\s]([a-z\d])(\w*)/g, -# function replacer(m, p1, p2) { -# return p1.toUpperCase() + p2; -# } -# ); -# }; - -# class Foo { -# async bar() {} -# } - -# class Cat { -# constructor(name) { -# this.name = name; -# } - -# speak() { -# console.log(this.name + ' makes a noise.'); -# } -# } - -# class Lion extends Cat { -# speak() { -# super.speak(); -# console.log(this.name + ' roars.'); -# } -# } -# """ - -# details = self.js_ts.get_program_details(source_file=code) -# self.assertEqual(len(details.classes), 3) -# self.assertEqual(len(details.callables), 5) - -# def test_get_classes(self): -# code = """ -# class Foo { -# async bar() {} -# } - -# class Cat { -# constructor(name) { -# this.name = name; -# } - -# speak() { -# console.log(this.name + ' makes a noise.'); -# } -# } - -# class Lion extends Cat { -# speak() { -# super.speak(); -# console.log(this.name + ' roars.'); -# } -# } -# """ - -# classes = self.js_ts.get_classes(code) -# self.assertEqual(len(classes), 3) - -# self.assertEqual(classes[0].name, "Foo") -# self.assertEqual(len(classes[0].methods), 1) -# self.assertEqual(classes[0].methods[0].name, "bar") - -# self.assertEqual(classes[1].name, "Cat") -# self.assertEqual(len(classes[1].methods), 2) -# self.assertEqual(classes[1].methods[0].name, "constructor") -# self.assertTrue(classes[1].methods[0].is_constructor) -# self.assertEqual(classes[1].methods[1].name, "speak") -# self.assertFalse(classes[1].methods[1].is_constructor) - -# self.assertEqual(classes[2].name, "Lion") -# self.assertEqual(len(classes[2].methods), 1) -# self.assertEqual(classes[2].methods[0].name, "speak") -# self.assertEqual(classes[2].parent, "Cat") - -# def test_get_all_callables(self): -# code = """ -# function test({b}, c = d, e = f, async = true) { -# } - -# export function *generateStuff(arg1, arg2) { -# yield arg1; -# } - -# async (a) => { return foo; } - -# export const -# a = function() { console.log("say hello")}, -# b = function() {}, -# yourFunctionName = () => console.log("say hello"); - -# c = str => 1, d = g => "a" -# """ - -# callables = self.js_ts.get_all_functions(code=code) -# self.assertEqual(len(callables), 7) - -# callable: JsCallable = get_callable(callables, "test") -# self.assertEqual(callable.signature, "test({b}, c = d, e = f, async = true)") - -# callable: JsCallable = get_callable(callables, "generateStuff") -# self.assertEqual(callable.signature, "generateStuff(arg1, arg2)") - -# callable: JsCallable = get_callable(callables, "a") -# self.assertEqual(callable.signature, "a()") - -# callable: JsCallable = get_callable(callables, "b") -# self.assertEqual(callable.signature, "b()") - -# callable: JsCallable = get_callable(callables, "yourFunctionName") -# self.assertEqual(callable.signature, "yourFunctionName()") - -# callable: JsCallable = get_callable(callables, "c") -# self.assertEqual(callable.signature, "c(str)") - -# callable: JsCallable = get_callable(callables, "d") -# self.assertEqual(callable.signature, "d(g)") - -# def test_get_top_level_functions(self): -# code = """ -# function test({b}, c = d, e = f, async = true) { -# } - -# export function exported_func(arg1, arg2) { -# } -# """ - -# callables = self.js_ts.get_all_functions(code=code) -# self.assertEqual(len(callables), 2) - -# callable: JsCallable = get_callable(callables, "test") -# self.assertEqual(callable.signature, "test({b}, c = d, e = f, async = true)") -# self.assertFalse(callable.is_constructor) - -# callable: JsCallable = get_callable(callables, "exported_func") -# self.assertEqual(callable.signature, "exported_func(arg1, arg2)") -# self.assertFalse(callable.is_constructor) - -# def test_get_top_level_generators(self): -# code = """ -# function *test({b}, c = d, e = f, async = true) { -# yield c; -# } - -# export function *exported_func(arg1, arg2) { -# } -# """ - -# callables = self.js_ts.get_all_functions(code=code) -# self.assertEqual(len(callables), 2) - -# callable: JsCallable = get_callable(callables, "test") -# self.assertEqual(callable.signature, "test({b}, c = d, e = f, async = true)") -# self.assertFalse(callable.is_constructor) - -# callable: JsCallable = get_callable(callables, "exported_func") -# self.assertEqual(callable.signature, "exported_func(arg1, arg2)") -# self.assertFalse(callable.is_constructor) - -# def test_get_top_level_arrow_functions(self): -# code = """ -# const toCamelCase = str => { -# return str.toLowerCase().replace(/[-_\s]([a-z\d])(\w*)/g, -# function replacer(m, p1, p2) { -# return p1.toUpperCase() + p2; -# } -# ); -# }; - -# const matchAll = (regExp, str) => { -# let matches; -# const arr = []; - -# while ((matches = regExp.exec(str)) !== null) { -# arr.push(matches); -# } - -# return arr; -# } - -# export const a = () => "", b = () => 12 - -# c = str => 1, d = g => "a" -# e = a => 12 - -# export default () => console.log("say hello"); -# """ - -# callables = self.js_ts.get_all_functions(code=code) -# self.assertEqual(len(callables), 8) - -# callable: JsCallable = get_callable(callables, "toCamelCase") -# self.assertEqual(callable.signature, "toCamelCase(str)") -# self.assertFalse(callable.is_constructor) - -# callable: JsCallable = get_callable(callables, "matchAll") -# self.assertEqual(callable.signature, "matchAll(regExp, str)") -# self.assertFalse(callable.is_constructor) - -# callable: JsCallable = get_callable(callables, "a") -# self.assertEqual(callable.signature, "a()") -# self.assertFalse(callable.is_constructor) - -# callable: JsCallable = get_callable(callables, "b") -# self.assertEqual(callable.signature, "b()") -# self.assertFalse(callable.is_constructor) - -# callable: JsCallable = get_callable(callables, "c") -# self.assertEqual(callable.signature, "c(str)") -# self.assertFalse(callable.is_constructor) - -# callable: JsCallable = get_callable(callables, "d") -# self.assertEqual(callable.signature, "d(g)") -# self.assertFalse(callable.is_constructor) - -# callable: JsCallable = get_callable(callables, "e") -# self.assertEqual(callable.signature, "e(a)") -# self.assertFalse(callable.is_constructor) - -# callable: JsCallable = get_callable(callables, "default") -# self.assertEqual(callable.signature, "default()") -# self.assertFalse(callable.is_constructor) - -# def test_get_top_level_function_expressions(self): -# code = """ -# abc = function() {}; - -# const func = function() {}; - -# export const bb = function(x, y) {}; - -# a = function(x) {}, b = function(y, z) {}; - -# export default function(w) { console.log("say hello")}; -# """ - -# callables = self.js_ts.get_all_functions(code=code) -# self.assertEqual(len(callables), 6) - -# callable: JsCallable = get_callable(callables, "abc") -# self.assertEqual(callable.signature, "abc()") -# self.assertFalse(callable.is_constructor) - -# callable: JsCallable = get_callable(callables, "func") -# self.assertEqual(callable.signature, "func()") -# self.assertFalse(callable.is_constructor) - -# callable: JsCallable = get_callable(callables, "bb") -# self.assertEqual(callable.signature, "bb(x, y)") -# self.assertFalse(callable.is_constructor) - -# callable: JsCallable = get_callable(callables, "b") -# self.assertEqual(callable.signature, "b(y, z)") -# self.assertFalse(callable.is_constructor) - -# callable: JsCallable = get_callable(callables, "a") -# self.assertEqual(callable.signature, "a(x)") -# self.assertFalse(callable.is_constructor) - -# callable: JsCallable = get_callable(callables, "default") -# self.assertEqual(callable.signature, "default(w)") -# self.assertFalse(callable.is_constructor) - -# def test_get_top_level_generator_expressions(self): -# code = """ -# abc = function *() {}; - -# const func = function *(x, y) {}; - -# export const bb = function*(x, y) {}; - -# a = function*(x) {}, b = function*(y, z) {}; - -# export default function *() {}; -# """ - -# callables = self.js_ts.get_all_functions(code=code) -# self.assertEqual(len(callables), 6) - -# callable: JsCallable = get_callable(callables, "abc") -# self.assertEqual(callable.signature, "abc()") -# self.assertFalse(callable.is_constructor) - -# callable: JsCallable = get_callable(callables, "func") -# self.assertEqual(callable.signature, "func(x, y)") -# self.assertFalse(callable.is_constructor) - -# callable: JsCallable = get_callable(callables, "bb") -# self.assertEqual(callable.signature, "bb(x, y)") -# self.assertFalse(callable.is_constructor) - -# callable: JsCallable = get_callable(callables, "b") -# self.assertEqual(callable.signature, "b(y, z)") -# self.assertFalse(callable.is_constructor) - -# callable: JsCallable = get_callable(callables, "a") -# self.assertEqual(callable.signature, "a(x)") -# self.assertFalse(callable.is_constructor) - -# callable: JsCallable = get_callable(callables, "default") -# self.assertEqual(callable.signature, "default()") -# self.assertFalse(callable.is_constructor) - - -# def get_callable(callables: List[JsCallable], name: str) -> JsCallable: -# for callable in callables: -# if callable.name == name: -# return callable - -# raise ValueError diff --git a/tests/tree_sitter/python/__init__.py b/tests/tree_sitter/python/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/tree_sitter/python/test_python_tree_sitter.py b/tests/tree_sitter/python/test_python_tree_sitter.py deleted file mode 100644 index 9f8d932..0000000 --- a/tests/tree_sitter/python/test_python_tree_sitter.py +++ /dev/null @@ -1,177 +0,0 @@ -# from unittest import TestCase - -# from cldk.analysis.python.treesitter import PythonSitter - - -# class TestPythonTreeSitter(TestCase): -# """ -# Tests for Python TreeSitter nodule -# """ - -# def setUp(self): -# """Runs before each test case""" -# self.python_tree_sitter = PythonSitter() - -# def tearDown(self): -# """Runs after each test case""" - -# def test_is_parasable(self): -# module_str = """ -# @staticmethod -# def foo() -> None: -# pass -# class Person: -# def __init__(self, name: str, age: int): -# self.name = name -# self.age = age -# @staticmethod -# def __str__(self):" -# """ -# self.assertFalse(self.python_tree_sitter.is_parsable(module_str)) - -# def test_get_all_methods(self): -# module_str = """ -# @staticmethod -# def foo() -> None: -# pass -# class Person: -# def __init__(self, name: str, age: int): -# self.name = name -# self.age = age -# @staticmethod -# def __str__(self): -# return f"{self.name}({self.age})" -# """ -# all_methods = self.python_tree_sitter.get_all_methods(module=module_str) -# all_functions = self.python_tree_sitter.get_all_functions(module=module_str) -# self.assertEquals(len(all_methods), 2) -# self.assertEquals(len(all_functions), 1) -# self.assertEquals(all_methods[0].full_signature, "__init__(self, name: str, age: int)") -# self.assertTrue(all_methods[0].is_constructor) -# self.assertFalse(all_methods[0].is_static) -# self.assertEquals(all_methods[0].class_signature, "Person") -# self.assertEquals(all_functions[0].class_signature, "") -# self.assertFalse(all_functions[0].is_static) - -# def test_get_all_imports(self): -# module_str = """ -# from typing import List - -# from sphinx.domains.python import PyField -# from tree_sitter import Language, Parser, Query, Node -# import a -# @staticmethod -# def foo() -> None: -# pass -# class Person: -# def __init__(self, name: str, age: int): -# self.name = name -# self.age = age -# @staticmethod -# def __str__(self): -# return f"{self.name}({self.age})" -# """ -# all_imports = self.python_tree_sitter.get_all_imports(module=module_str) -# self.assertEquals(len(all_imports), 4) - -# def test_get_all_imports_details(self): -# module_str = """ -# from typing import List - -# from sphinx.domains.python import PyField -# from tree_sitter import Language, Parser, Query, Node -# import a -# @staticmethod -# def foo() -> None: -# pass -# class Person: -# def __init__(self, name: str, age: int): -# self.name = name -# self.age = age -# @staticmethod -# def __str__(self): -# return f"{self.name}({self.age})" -# """ -# all_imports = self.python_tree_sitter.get_all_imports_details(module=module_str) -# self.assertEquals(len(all_imports), 4) - -# def test_get_module(self): -# module_str = """ -# from typing import List - -# from sphinx.domains.python import PyField -# from tree_sitter import Language, Parser, Query, Node -# import a -# @staticmethod -# def foo() -> None: -# pass -# class Person: -# def __init__(self, name: str, age: int): -# self.name = name -# self.age = age -# @staticmethod -# def __str__(self): -# return f"{self.name}({self.age})" -# """ -# module_details = self.python_tree_sitter.get_module_details(module=module_str) -# self.assertIsNotNone(module_details.functions) -# self.assertIsNotNone(module_details.classes) -# self.assertIsNotNone(module_details.imports) - -# def test_get_all_classes(self): -# module_str = """ -# def foo() -> None: -# pass -# class Person: -# def __init__(self, name: str, age: int): -# self.name = name -# self.age = age - -# def __str__(self): -# return f"{self.name}({self.age})" -# """ -# all_classes = self.python_tree_sitter.get_all_classes(module=module_str) -# self.assertEquals(len(all_classes), 1) -# self.assertEquals(len(all_classes[0].methods), 2) -# self.assertEquals(all_classes[0].methods[0].full_signature, "__init__(self, name: str, age: int)") -# self.assertEquals(all_classes[0].methods[1].class_signature, "Person") - -# def test_get_all_test_classes(self): -# module_str = """ -# import unittest - -# class TestStringMethods(unittest.TestCase, ABC): - -# def test_upper(self): -# self.assertEqual('foo'.upper(), 'FOO') - -# def test_isupper(self): -# self.assertTrue('FOO'.isupper()) -# self.assertFalse('Foo'.isupper()) - -# def test_split(self): -# s = 'hello world' -# self.assertEqual(s.split(), ['hello', 'world']) -# # check that s.split fails when the separator is not a string -# with self.assertRaises(TypeError): -# s.split(2) - -# if __name__ == '__main__': -# unittest.main() -# """ -# all_classes = self.python_tree_sitter.get_all_classes(module=module_str) -# self.assertTrue(all_classes[0].is_test_class) - -# def test_call_site(self): -# module_str = """ -# import unittest - -# class TestStringMethods(unittest.TestCase, ABC): - -# def test_get_all_test_classes(self): -# module_str = "a" -# all_classes = self.python_tree_sitter.get_all_classes(module=module_str) -# self.assertTrue(all_classes[0].is_test_class) -# """ -# all_classes = self.python_tree_sitter.get_all_classes(module=module_str) -# self.assertTrue(all_classes[0].is_test_class) From 32da3589f8f075d5f2db83901a241b75e01b20e2 Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Wed, 13 Nov 2024 13:59:32 -0500 Subject: [PATCH 2/4] Update poetry build rules to include codeanalyzer-*.jar and add a test case to verify it exists. Signed-off-by: Rahul Krishna --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 346b2f7..6acc889 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "cldk" -version = "0.3.0" +version = "0.4.0" description = "codellm-devkit: A python library for seamless integration with LLMs." authors = ["Rahul Krishna ", "Rangeet Pan ", "Saurabh Sinhas ", "Raju Pavuluri "] From a7362bf685726e207eb454103825be6cfb8c5eec Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Wed, 13 Nov 2024 13:59:53 -0500 Subject: [PATCH 3/4] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 346b2f7..6acc889 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "cldk" -version = "0.3.0" +version = "0.4.0" description = "codellm-devkit: A python library for seamless integration with LLMs." authors = ["Rahul Krishna ", "Rangeet Pan ", "Saurabh Sinhas ", "Raju Pavuluri "] From 8a2b1d5a2f627e5859c25345dfc114c8490b8fdb Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Wed, 13 Nov 2024 14:58:40 -0500 Subject: [PATCH 4/4] Fix issue 67--symbol table is none. Signed-off-by: Rahul Krishna --- cldk/analysis/java/codeanalyzer/codeanalyzer.py | 2 +- cldk/models/java/models.py | 6 ++++-- tests/analysis/java/test_java.py | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/cldk/analysis/java/codeanalyzer/codeanalyzer.py b/cldk/analysis/java/codeanalyzer/codeanalyzer.py index 8bd3e17..83562a7 100644 --- a/cldk/analysis/java/codeanalyzer/codeanalyzer.py +++ b/cldk/analysis/java/codeanalyzer/codeanalyzer.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. ################################################################################ - +from pdb import set_trace import re import json import shlex diff --git a/cldk/models/java/models.py b/cldk/models/java/models.py index c156268..2579bb8 100644 --- a/cldk/models/java/models.py +++ b/cldk/models/java/models.py @@ -20,7 +20,7 @@ import re from contextvars import ContextVar from typing import Dict, List, Optional - +from pdb import set_trace from pydantic import BaseModel, field_validator, model_validator from .constants_namespace import ConstantsNamespace @@ -412,9 +412,11 @@ class JApplication(BaseModel): @field_validator("symbol_table", mode="after") @classmethod - def validate_source(cls, symbol_table): + def validate_source(cls, symbol_table) -> Dict[str, JCompilationUnit]: # Populate the lookup table for callables for _, j_compulation_unit in symbol_table.items(): for type_declaration, jtype in j_compulation_unit.type_declarations.items(): for __, j_callable in jtype.callable_declarations.items(): _CALLABLES_LOOKUP_TABLE[(type_declaration, j_callable.signature)] = j_callable + + return symbol_table diff --git a/tests/analysis/java/test_java.py b/tests/analysis/java/test_java.py index 839680f..b7ff41b 100644 --- a/tests/analysis/java/test_java.py +++ b/tests/analysis/java/test_java.py @@ -1,9 +1,23 @@ +from pdb import set_trace from cldk import CLDK from typing import List, Tuple from cldk.analysis import AnalysisLevel from cldk.models.java.models import JMethodDetail +def test_get_symbol_table_is_not_null(test_fixture, codeanalyzer_jar_path): + # Initialize the CLDK object with the project directory, language, and analysis_backend. + cldk = CLDK(language="java") + analysis = cldk.analysis( + project_path=test_fixture, + analysis_backend="codeanalyzer", + analysis_backend_path=codeanalyzer_jar_path, + eager=True, + analysis_level=AnalysisLevel.call_graph, + ) + assert analysis.get_symbol_table() is not None + + def test_get_class_call_graph(test_fixture, codeanalyzer_jar_path): # Initialize the CLDK object with the project directory, language, and analysis_backend. cldk = CLDK(language="java")