From 95506acdd39ad7f7ef3132c4f81e9936e114cef1 Mon Sep 17 00:00:00 2001 From: dev server Date: Thu, 15 Sep 2022 19:35:35 +0200 Subject: [PATCH 01/10] Raise versions to 0.5.0-SNAPSHOT --- java/build.gradle | 4 ++-- java/src/main/java/tftest/App.java | 29 ++--------------------------- python/requirements.txt | 2 +- python/runPython.sh | 2 +- 4 files changed, 6 insertions(+), 31 deletions(-) diff --git a/java/build.gradle b/java/build.gradle index 04ab722..f6d2958 100644 --- a/java/build.gradle +++ b/java/build.gradle @@ -18,9 +18,9 @@ repositories { } dependencies { - implementation group: 'org.tensorflow', name: 'tensorflow-core-platform', version: '0.4.1' + //implementation group: 'org.tensorflow', name: 'tensorflow-core-platform', version: '0.4.1' // we need 0.5.0-SNAPSHOT with TF2.9.1, the 0.4.1 version has TF 2.4 - //implementation group: 'org.tensorflow', name: 'tensorflow-core-platform', version: '0.5.0-SNAPSHOT' + implementation group: 'org.tensorflow', name: 'tensorflow-core-platform', version: '0.5.0-SNAPSHOT' } diff --git a/java/src/main/java/tftest/App.java b/java/src/main/java/tftest/App.java index 1f87240..89665ce 100644 --- a/java/src/main/java/tftest/App.java +++ b/java/src/main/java/tftest/App.java @@ -21,32 +21,7 @@ public static void main(String[] args) { System.out.println("Testing query without ConfigProto"); try (SavedModelBundle savedModel = SavedModelBundle.loader(modelPath).withTags(new String[]{"serve"}).load()) { doInference(savedModel, "Java: Model without ConfigProto"); - } - System.out.println("Testing query with ConfigProto"); - ConfigProto config = ConfigProto.newBuilder(ConfigProto.getDefaultInstance()) - .setLogDevicePlacement(false) - .setGraphOptions(GraphOptions.newBuilder() - .setOptimizerOptions( - OptimizerOptions.newBuilder() - .setCpuGlobalJit(false) - .setGlobalJitLevel(GlobalJitLevel.OFF) - .setDoCommonSubexpressionElimination(false) - .setDoConstantFolding(false) - .setDoFunctionInlining(false) - .setOptLevel(Level.L0) - .build() - ) - ).setGpuOptions( - GPUOptions.newBuilder() - .setForceGpuCompatible(false) - .setAllowGrowth(true) - .setPerProcessGpuMemoryFraction(0.5) - .build() - ).build(); - try (SavedModelBundle savedModel = SavedModelBundle.loader(modelPath).withConfigProto(config).withTags(new String[]{"serve"}).load()) { - doInference(savedModel, "Java: Model with ConfigProto/disabled JIT"); - } - + } } public static void doInference(SavedModelBundle savedModel, String msg) { @@ -54,7 +29,7 @@ public static void doInference(SavedModelBundle savedModel, String msg) { try (TFloat32 xTensor = TFloat32.tensorOf(NdArrays.ofFloats(Shape.of(1,244,244,3))); TFloat32 zTensor = (TFloat32) savedModel .call(Collections.singletonMap("inputs", xTensor)) - .get("output_0")) { + .get("output_0").get()) { long end = System.currentTimeMillis(); System.out.println(msg + ", warm up took "+((end-start)/1000f)+" seconds"); diff --git a/python/requirements.txt b/python/requirements.txt index 42c194e..8fd6099 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,2 +1,2 @@ -tensorflow==2.7.1 +tensorflow==2.9.1 tensorflow-hub==0.12.0 diff --git a/python/runPython.sh b/python/runPython.sh index f3101d5..e3d3227 100755 --- a/python/runPython.sh +++ b/python/runPython.sh @@ -1,4 +1,4 @@ #!/bin/bash cd "$(dirname "$BASH_SOURCE")" -docker run -it -w /work -v "$(pwd):/work" -v "/tmp/vit_b32_fe/:/tmp/vit_b32_fe/" tensorflow/tensorflow:2.7.1 bash -c "pip3 install -r requirements.txt && python3 main.py" +docker run -it -w /work -v "$(pwd):/work" -v "/tmp/vit_b32_fe/:/tmp/vit_b32_fe/" tensorflow/tensorflow:2.9.1 bash -c "pip3 install -r requirements.txt && python3 main.py" From a150aa689d1e0a6ca3e91e8583b6d1f3349c30a0 Mon Sep 17 00:00:00 2001 From: dev server Date: Thu, 15 Sep 2022 19:48:44 +0200 Subject: [PATCH 02/10] Updated to 0.5.0-SNAPSHOT --- README.md | 147 +----------------------------------------------------- 1 file changed, 1 insertion(+), 146 deletions(-) diff --git a/README.md b/README.md index 62c045b..2e0935c 100644 --- a/README.md +++ b/README.md @@ -1,154 +1,9 @@ # Tensorflow java test -This repo contains code to reproduce the issue [#437](https://github.com/tensorflow/java/issues/473) in [tensorflow java bindings repository](https://github.com/tensorflow/) in version 0.4.1. +This repo contains code to reproduce the issue [#437](https://github.com/tensorflow/java/issues/473) in [tensorflow java bindings repository](https://github.com/tensorflow/) in version 0.5.0-SNAPSHOT. ## Usage Make sure a recent version of docker is installed. Tested with Ubuntu 22.04 on x86_64. The Python and Java code is executed in docker containers to have a reproducable environment. -### Run all at once -``` -phoenix@dev:~/workspaces/research/tf_tests$ ./run.sh 2> /dev/null | grep "warm up" -Python: warm up finished in 22.75354215799598 seconds -Java: Model without ConfigProto, warm up took 23.337 seconds -Java: Model with ConfigProto/disabled JIT, warm up took 23.55 seconds -``` - - -### Run each test (python, java) individually - * Model download -``` -phoenix@dev:~/workspaces/research/tf_tests$ ./run.sh only_dl -Download model . . . ---2022-09-15 18:25:20-- https://tfhub.dev/sayakpaul/vit_b32_fe/1?tf-hub-format=compressed -Auflösen des Hostnamens tfhub.dev (tfhub.dev) … 142.250.186.174, 2a00:1450:4001:82b::200e -Verbindungsaufbau zu tfhub.dev (tfhub.dev)|142.250.186.174|:443 … verbunden. -HTTP-Anforderung gesendet, auf Antwort wird gewartet … 302 Found -Platz: https://storage.googleapis.com/tfhub-modules/sayakpaul/vit_b32_fe/1.tar.gz [folgend] ---2022-09-15 18:25:20-- https://storage.googleapis.com/tfhub-modules/sayakpaul/vit_b32_fe/1.tar.gz -Auflösen des Hostnamens storage.googleapis.com (storage.googleapis.com) … 142.250.185.240, 216.58.212.176, 142.250.74.208, ... -Verbindungsaufbau zu storage.googleapis.com (storage.googleapis.com)|142.250.185.240|:443 … verbunden. -HTTP-Anforderung gesendet, auf Antwort wird gewartet … 200 OK -Länge: 324615541 (310M) [application/octet-stream] -Wird in ‘/tmp/vit_b32_fe/vit_b32_fe_1.tar.gz’ gespeichert. - -/tmp/vit_b32_fe/vit_b32_fe_1.tar.gz 100%[=====================================================================================>] 309,58M 23,4MB/s in 13s - -2022-09-15 18:25:34 (23,7 MB/s) - ‘/tmp/vit_b32_fe/vit_b32_fe_1.tar.gz’ gespeichert [324615541/324615541] - -Extract model . . . -Model can be loaded from /tmp/vit_b32_fe/ -``` - * Python test -``` -hoenix@dev:~/workspaces/research/tf_tests$ ./python/runPython.sh -Collecting tensorflow==2.7.1 - Downloading tensorflow-2.7.1-cp38-cp38-manylinux2010_x86_64.whl (495.1 MB) - |████████████████████████████████| 495.1 MB 43 kB/s -Collecting tensorflow-hub==0.12.0 - Downloading tensorflow_hub-0.12.0-py2.py3-none-any.whl (108 kB) - |████████████████████████████████| 108 kB 25.6 MB/s -Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.7.1->-r requirements.txt (line 1)) (4.0.1) -Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.7.1->-r requirements.txt (line 1)) (1.6.3) -Requirement already satisfied: flatbuffers<3.0,>=1.12 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.7.1->-r requirements.txt (line 1)) (2.0) -Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.21.0 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.7.1->-r requirements.txt (line 1)) (0.23.1) -Requirement already satisfied: keras<2.8,>=2.7.0rc0 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.7.1->-r requirements.txt (line 1)) (2.7.0) -Requirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.7.1->-r requirements.txt (line 1)) (3.19.4) -Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.7.1->-r requirements.txt (line 1)) (3.6.0) -Requirement already satisfied: numpy>=1.14.5 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.7.1->-r requirements.txt (line 1)) (1.22.1) -Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.7.1->-r requirements.txt (line 1)) (3.3.0) -Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.7.1->-r requirements.txt (line 1)) (1.1.0) -Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.7.1->-r requirements.txt (line 1)) (1.43.0) -Requirement already satisfied: gast<0.5.0,>=0.2.1 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.7.1->-r requirements.txt (line 1)) (0.4.0) -Requirement already satisfied: tensorboard~=2.6 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.7.1->-r requirements.txt (line 1)) (2.8.0) -Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.7.1->-r requirements.txt (line 1)) (0.2.0) -Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.7.1->-r requirements.txt (line 1)) (1.13.3) -Requirement already satisfied: keras-preprocessing>=1.1.1 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.7.1->-r requirements.txt (line 1)) (1.1.2) -Requirement already satisfied: tensorflow-estimator<2.8,~=2.7.0rc0 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.7.1->-r requirements.txt (line 1)) (2.7.0) -Requirement already satisfied: absl-py>=0.4.0 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.7.1->-r requirements.txt (line 1)) (1.0.0) -Requirement already satisfied: wheel<1.0,>=0.32.0 in /usr/lib/python3/dist-packages (from tensorflow==2.7.1->-r requirements.txt (line 1)) (0.34.2) -Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.7.1->-r requirements.txt (line 1)) (1.16.0) -Requirement already satisfied: libclang>=9.0.1 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.7.1->-r requirements.txt (line 1)) (13.0.0) -Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.8/dist-packages (from tensorboard~=2.6->tensorflow==2.7.1->-r requirements.txt (line 1)) (0.6.1) -Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.8/dist-packages (from tensorboard~=2.6->tensorflow==2.7.1->-r requirements.txt (line 1)) (3.3.6) -Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.8/dist-packages (from tensorboard~=2.6->tensorflow==2.7.1->-r requirements.txt (line 1)) (2.6.0) -Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.8/dist-packages (from tensorboard~=2.6->tensorflow==2.7.1->-r requirements.txt (line 1)) (2.27.1) -Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.8/dist-packages (from tensorboard~=2.6->tensorflow==2.7.1->-r requirements.txt (line 1)) (2.0.2) -Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.8/dist-packages (from tensorboard~=2.6->tensorflow==2.7.1->-r requirements.txt (line 1)) (0.4.6) -Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.8/dist-packages (from tensorboard~=2.6->tensorflow==2.7.1->-r requirements.txt (line 1)) (1.8.1) -Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.8/dist-packages (from tensorboard~=2.6->tensorflow==2.7.1->-r requirements.txt (line 1)) (60.7.0) -Requirement already satisfied: importlib-metadata>=4.4; python_version < "3.10" in /usr/local/lib/python3.8/dist-packages (from markdown>=2.6.8->tensorboard~=2.6->tensorflow==2.7.1->-r requirements.txt (line 1)) (4.10.1) -Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.8/dist-packages (from google-auth<3,>=1.6.3->tensorboard~=2.6->tensorflow==2.7.1->-r requirements.txt (line 1)) (5.0.0) -Requirement already satisfied: rsa<5,>=3.1.4; python_version >= "3.6" in /usr/local/lib/python3.8/dist-packages (from google-auth<3,>=1.6.3->tensorboard~=2.6->tensorflow==2.7.1->-r requirements.txt (line 1)) (4.8) -Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.8/dist-packages (from google-auth<3,>=1.6.3->tensorboard~=2.6->tensorflow==2.7.1->-r requirements.txt (line 1)) (0.2.8) -Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow==2.7.1->-r requirements.txt (line 1)) (1.26.8) -Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow==2.7.1->-r requirements.txt (line 1)) (2021.10.8) -Requirement already satisfied: charset-normalizer~=2.0.0; python_version >= "3" in /usr/local/lib/python3.8/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow==2.7.1->-r requirements.txt (line 1)) (2.0.11) -Requirement already satisfied: idna<4,>=2.5; python_version >= "3" in /usr/local/lib/python3.8/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.6->tensorflow==2.7.1->-r requirements.txt (line 1)) (3.3) -Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.8/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.6->tensorflow==2.7.1->-r requirements.txt (line 1)) (1.3.1) -Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.8/dist-packages (from importlib-metadata>=4.4; python_version < "3.10"->markdown>=2.6.8->tensorboard~=2.6->tensorflow==2.7.1->-r requirements.txt (line 1)) (3.7.0) -Requirement already satisfied: pyasn1>=0.1.3 in /usr/local/lib/python3.8/dist-packages (from rsa<5,>=3.1.4; python_version >= "3.6"->google-auth<3,>=1.6.3->tensorboard~=2.6->tensorflow==2.7.1->-r requirements.txt (line 1)) (0.4.8) -Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.8/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.6->tensorflow==2.7.1->-r requirements.txt (line 1)) (3.2.0) -Installing collected packages: tensorflow, tensorflow-hub -Successfully installed tensorflow-2.7.1 tensorflow-hub-0.12.0 -WARNING: You are using pip version 20.2.4; however, version 22.2.2 is available. -You should consider upgrading via the '/usr/bin/python3 -m pip install --upgrade pip' command. -2022-09-15 16:26:29.533259: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory -2022-09-15 16:26:29.533274: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. -Loading saved model from /tmp/vit_b32_fe/ -2022-09-15 16:26:30.297888: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory -2022-09-15 16:26:30.297903: W tensorflow/stream_executor/cuda/cuda_driver.cc:269] failed call to cuInit: UNKNOWN ERROR (303) -2022-09-15 16:26:30.297918: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:163] no NVIDIA GPU device is present: /dev/nvidia0 does not exist -2022-09-15 16:26:30.298052: I tensorflow/core/platform/cpu_feature_guard.cc:151] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA -To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. -Python: warm up finished in 22.75354215799598 seconds -``` - * Java test -``` -phoenix@dev:~/workspaces/research/tf_tests$ ./java/runJava.sh - -Welcome to Gradle 7.5.1! - -Here are the highlights of this release: - - Support for Java 18 - - Support for building with Groovy 4 - - Much more responsive continuous builds - - Improved diagnostics for dependency resolution - -For more details see https://docs.gradle.org/7.5.1/release-notes.html - -Starting a Gradle Daemon (subsequent builds will be faster) - -> Task :run -Testing query without ConfigProto -2022-09-15 16:27:06.411118: I external/org_tensorflow/tensorflow/cc/saved_model/reader.cc:43] Reading SavedModel from: /tmp/vit_b32_fe/ -2022-09-15 16:27:06.446895: I external/org_tensorflow/tensorflow/cc/saved_model/reader.cc:107] Reading meta graph with tags { serve } -2022-09-15 16:27:06.446917: I external/org_tensorflow/tensorflow/cc/saved_model/reader.cc:148] Reading SavedModel debug info (if present) from: /tmp/vit_b32_fe/ -2022-09-15 16:27:06.446951: I external/org_tensorflow/tensorflow/core/platform/cpu_feature_guard.cc:151] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA -To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. -2022-09-15 16:27:06.581217: I external/org_tensorflow/tensorflow/cc/saved_model/loader.cc:228] Restoring SavedModel bundle. -2022-09-15 16:27:07.036773: I external/org_tensorflow/tensorflow/cc/saved_model/loader.cc:212] Running initialization op on SavedModel bundle at path: /tmp/vit_b32_fe/ -2022-09-15 16:27:07.243972: I external/org_tensorflow/tensorflow/cc/saved_model/loader.cc:301] SavedModel load for tags { serve }; Status: success: OK. Took 832857 microseconds. -TF_XLA_FLAGS not set -2022-09-15 16:27:08.349033: I external/org_tensorflow/tensorflow/compiler/xla/service/service.cc:171] XLA service 0x7fae1407d3d0 initialized for platform Host (this does not guarantee that XLA will be used). Devices: -2022-09-15 16:27:08.349064: I external/org_tensorflow/tensorflow/compiler/xla/service/service.cc:179] StreamExecutor device (0): Host, Default Version -2022-09-15 16:27:08.381484: I external/org_tensorflow/tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:237] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable. -2022-09-15 16:27:30.690163: I external/org_tensorflow/tensorflow/compiler/jit/xla_compilation_cache.cc:351] Compiled cluster using XLA! This line is logged at most once for the lifetime of the process. -Java: Model without ConfigProto, warm up took 23.241 seconds -Testing query with ConfigProto -2022-09-15 16:27:30.867283: I external/org_tensorflow/tensorflow/cc/saved_model/reader.cc:43] Reading SavedModel from: /tmp/vit_b32_fe/ -2022-09-15 16:27:30.892289: I external/org_tensorflow/tensorflow/cc/saved_model/reader.cc:107] Reading meta graph with tags { serve } -2022-09-15 16:27:30.892315: I external/org_tensorflow/tensorflow/cc/saved_model/reader.cc:148] Reading SavedModel debug info (if present) from: /tmp/vit_b32_fe/ -2022-09-15 16:27:30.982714: I external/org_tensorflow/tensorflow/cc/saved_model/loader.cc:228] Restoring SavedModel bundle. -2022-09-15 16:27:31.400043: I external/org_tensorflow/tensorflow/cc/saved_model/loader.cc:212] Running initialization op on SavedModel bundle at path: /tmp/vit_b32_fe/ -2022-09-15 16:27:31.609886: I external/org_tensorflow/tensorflow/cc/saved_model/loader.cc:301] SavedModel load for tags { serve }; Status: success: OK. Took 742610 microseconds. -TF_XLA_FLAGS not set -Java: Model with ConfigProto/disabled JIT, warm up took 23.505 seconds - -BUILD SUCCESSFUL in 1m 4s -2 actionable tasks: 1 executed, 1 up-to-date -``` - - - From 5f47cd1aeac5a4fcc173fd327588902858b203cd Mon Sep 17 00:00:00 2001 From: Sebastian Lutter Date: Fri, 16 Sep 2022 11:23:01 +0200 Subject: [PATCH 03/10] Results of 0.5.0-SNAPSHOT (TF 2.9.1) added --- README.md | 140 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) diff --git a/README.md b/README.md index 2e0935c..9535a35 100644 --- a/README.md +++ b/README.md @@ -7,3 +7,143 @@ This repo contains code to reproduce the issue [#437](https://github.com/tensorf Make sure a recent version of docker is installed. Tested with Ubuntu 22.04 on x86_64. The Python and Java code is executed in docker containers to have a reproducable environment. +### Run all at once +``` +phoenix@dev:~/workspaces/research/tf_tests$ ./run.sh 2> /dev/null | grep "warm up" +Python: warm up finished in 4.694977925988496 seconds +Java: Model without ConfigProto, warm up took 6.797 seconds +``` + +* Model download +``` +phoenix@dev:~/workspaces/research/tf_tests$ ./run.sh only_dl +Download model . . . +--2022-09-15 18:25:20-- https://tfhub.dev/sayakpaul/vit_b32_fe/1?tf-hub-format=compressed +Auflösen des Hostnamens tfhub.dev (tfhub.dev) … 142.250.186.174, 2a00:1450:4001:82b::200e +Verbindungsaufbau zu tfhub.dev (tfhub.dev)|142.250.186.174|:443 … verbunden. +HTTP-Anforderung gesendet, auf Antwort wird gewartet … 302 Found +Platz: https://storage.googleapis.com/tfhub-modules/sayakpaul/vit_b32_fe/1.tar.gz [folgend] +--2022-09-15 18:25:20-- https://storage.googleapis.com/tfhub-modules/sayakpaul/vit_b32_fe/1.tar.gz +Auflösen des Hostnamens storage.googleapis.com (storage.googleapis.com) … 142.250.185.240, 216.58.212.176, 142.250.74.208, ... +Verbindungsaufbau zu storage.googleapis.com (storage.googleapis.com)|142.250.185.240|:443 … verbunden. +HTTP-Anforderung gesendet, auf Antwort wird gewartet … 200 OK +Länge: 324615541 (310M) [application/octet-stream] +Wird in ‘/tmp/vit_b32_fe/vit_b32_fe_1.tar.gz’ gespeichert. + +/tmp/vit_b32_fe/vit_b32_fe_1.tar.gz 100%[=====================================================================================>] 309,58M 23,4MB/s in 13s + +2022-09-15 18:25:34 (23,7 MB/s) - ‘/tmp/vit_b32_fe/vit_b32_fe_1.tar.gz’ gespeichert [324615541/324615541] + +Extract model . . . +Model can be loaded from /tmp/vit_b32_fe/ +``` + +* Python test +``` +phoenix@dev:~/workspaces/research/tf_tests$ ./python/runPython.sh +Collecting tensorflow==2.9.1 + Downloading tensorflow-2.9.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (511.7 MB) + |████████████████████████████████| 511.7 MB 26.2 MB/s +Collecting tensorflow-hub==0.12.0 + Downloading tensorflow_hub-0.12.0-py2.py3-none-any.whl (108 kB) + |████████████████████████████████| 108 kB 28.6 MB/s +Requirement already satisfied: tensorboard<2.10,>=2.9 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.9.1->-r requirements.txt (line 1)) (2.9.0) +Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.9.1->-r requirements.txt (line 1)) (4.2.0) +Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.9.1->-r requirements.txt (line 1)) (0.26.0) +Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.9.1->-r requirements.txt (line 1)) (3.3.0) +Requirement already satisfied: keras-preprocessing>=1.1.1 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.9.1->-r requirements.txt (line 1)) (1.1.2) +Requirement already satisfied: protobuf<3.20,>=3.9.2 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.9.1->-r requirements.txt (line 1)) (3.19.4) +Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.9.1->-r requirements.txt (line 1)) (1.1.0) +Requirement already satisfied: keras<2.10.0,>=2.9.0rc0 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.9.1->-r requirements.txt (line 1)) (2.9.0) +Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.9.1->-r requirements.txt (line 1)) (1.6.3) +Requirement already satisfied: gast<=0.4.0,>=0.2.1 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.9.1->-r requirements.txt (line 1)) (0.4.0) +Requirement already satisfied: flatbuffers<2,>=1.12 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.9.1->-r requirements.txt (line 1)) (1.12) +Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.9.1->-r requirements.txt (line 1)) (3.6.0) +Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.9.1->-r requirements.txt (line 1)) (1.46.3) +Requirement already satisfied: numpy>=1.20 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.9.1->-r requirements.txt (line 1)) (1.22.4) +Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.9.1->-r requirements.txt (line 1)) (1.16.0) +Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.9.1->-r requirements.txt (line 1)) (14.0.1) +Requirement already satisfied: setuptools in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.9.1->-r requirements.txt (line 1)) (62.3.2) +Requirement already satisfied: packaging in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.9.1->-r requirements.txt (line 1)) (21.3) +Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.9.1->-r requirements.txt (line 1)) (0.2.0) +Requirement already satisfied: tensorflow-estimator<2.10.0,>=2.9.0rc0 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.9.1->-r requirements.txt (line 1)) (2.9.0) +Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.9.1->-r requirements.txt (line 1)) (1.14.1) +Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.8/dist-packages (from tensorflow==2.9.1->-r requirements.txt (line 1)) (1.0.0) +Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.8/dist-packages (from tensorboard<2.10,>=2.9->tensorflow==2.9.1->-r requirements.txt (line 1)) (0.6.1) +Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.8/dist-packages (from tensorboard<2.10,>=2.9->tensorflow==2.9.1->-r requirements.txt (line 1)) (2.6.6) +Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.8/dist-packages (from tensorboard<2.10,>=2.9->tensorflow==2.9.1->-r requirements.txt (line 1)) (3.3.7) +Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.8/dist-packages (from tensorboard<2.10,>=2.9->tensorflow==2.9.1->-r requirements.txt (line 1)) (2.1.2) +Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.8/dist-packages (from tensorboard<2.10,>=2.9->tensorflow==2.9.1->-r requirements.txt (line 1)) (0.4.6) +Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.8/dist-packages (from tensorboard<2.10,>=2.9->tensorflow==2.9.1->-r requirements.txt (line 1)) (1.8.1) +Requirement already satisfied: wheel>=0.26 in /usr/lib/python3/dist-packages (from tensorboard<2.10,>=2.9->tensorflow==2.9.1->-r requirements.txt (line 1)) (0.34.2) +Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.8/dist-packages (from tensorboard<2.10,>=2.9->tensorflow==2.9.1->-r requirements.txt (line 1)) (2.27.1) +Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from packaging->tensorflow==2.9.1->-r requirements.txt (line 1)) (3.0.9) +Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.8/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow==2.9.1->-r requirements.txt (line 1)) (0.2.8) +Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.8/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow==2.9.1->-r requirements.txt (line 1)) (5.1.0) +Requirement already satisfied: rsa<5,>=3.1.4; python_version >= "3.6" in /usr/local/lib/python3.8/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow==2.9.1->-r requirements.txt (line 1)) (4.8) +Requirement already satisfied: importlib-metadata>=4.4; python_version < "3.10" in /usr/local/lib/python3.8/dist-packages (from markdown>=2.6.8->tensorboard<2.10,>=2.9->tensorflow==2.9.1->-r requirements.txt (line 1)) (4.11.4) +Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.8/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.10,>=2.9->tensorflow==2.9.1->-r requirements.txt (line 1)) (1.3.1) +Requirement already satisfied: charset-normalizer~=2.0.0; python_version >= "3" in /usr/local/lib/python3.8/dist-packages (from requests<3,>=2.21.0->tensorboard<2.10,>=2.9->tensorflow==2.9.1->-r requirements.txt (line 1)) (2.0.12) +Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests<3,>=2.21.0->tensorboard<2.10,>=2.9->tensorflow==2.9.1->-r requirements.txt (line 1)) (1.26.9) +Requirement already satisfied: idna<4,>=2.5; python_version >= "3" in /usr/local/lib/python3.8/dist-packages (from requests<3,>=2.21.0->tensorboard<2.10,>=2.9->tensorflow==2.9.1->-r requirements.txt (line 1)) (3.3) +Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests<3,>=2.21.0->tensorboard<2.10,>=2.9->tensorflow==2.9.1->-r requirements.txt (line 1)) (2022.5.18.1) +Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.8/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow==2.9.1->-r requirements.txt (line 1)) (0.4.8) +Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.8/dist-packages (from importlib-metadata>=4.4; python_version < "3.10"->markdown>=2.6.8->tensorboard<2.10,>=2.9->tensorflow==2.9.1->-r requirements.txt (line 1)) (3.8.0) +Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.8/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.10,>=2.9->tensorflow==2.9.1->-r requirements.txt (line 1)) (3.2.0) +Installing collected packages: tensorflow, tensorflow-hub +Successfully installed tensorflow-2.9.1 tensorflow-hub-0.12.0 +WARNING: You are using pip version 20.2.4; however, version 22.2.2 is available. +You should consider upgrading via the '/usr/bin/python3 -m pip install --upgrade pip' command. +2022-09-16 09:19:36.925258: I tensorflow/core/util/util.cc:169] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2022-09-16 09:19:36.929035: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory +2022-09-16 09:19:36.929046: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +Loading saved model from /tmp/vit_b32_fe/ +2022-09-16 09:19:37.986509: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory +2022-09-16 09:19:37.986530: W tensorflow/stream_executor/cuda/cuda_driver.cc:269] failed call to cuInit: UNKNOWN ERROR (303) +2022-09-16 09:19:37.986550: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (8e703ab9f298): /proc/driver/nvidia/version does not exist +2022-09-16 09:19:37.986689: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX_VNNI FMA +To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2022-09-16 09:19:40.909455: I tensorflow/compiler/xla/service/service.cc:170] XLA service 0x1cac60c0 initialized for platform Host (this does not guarantee that XLA will be used). Devices: +2022-09-16 09:19:40.909480: I tensorflow/compiler/xla/service/service.cc:178] StreamExecutor device (0): Host, Default Version +2022-09-16 09:19:41.005726: I tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:263] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable. +2022-09-16 09:19:45.697414: I tensorflow/compiler/jit/xla_compilation_cache.cc:478] Compiled cluster using XLA! This line is logged at most once for the lifetime of the process. +Python: warm up without flags finished in 4.988460940003279 seconds +``` + +* Java test +``` +phoenix@dev:~/workspaces/research/tf_tests$ ./java/runJava.sh + +Welcome to Gradle 7.5.1! + +Here are the highlights of this release: + - Support for Java 18 + - Support for building with Groovy 4 + - Much more responsive continuous builds + - Improved diagnostics for dependency resolution + +For more details see https://docs.gradle.org/7.5.1/release-notes.html + +Starting a Gradle Daemon (subsequent builds will be faster) + +> Task :run +Testing query without ConfigProto +2022-09-16 09:21:51.859288: I external/org_tensorflow/tensorflow/core/util/util.cc:169] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`. +2022-09-16 09:21:53.094691: I external/org_tensorflow/tensorflow/cc/saved_model/reader.cc:43] Reading SavedModel from: /tmp/vit_b32_fe/ +2022-09-16 09:21:53.139816: I external/org_tensorflow/tensorflow/cc/saved_model/reader.cc:81] Reading meta graph with tags { serve } +2022-09-16 09:21:53.139846: I external/org_tensorflow/tensorflow/cc/saved_model/reader.cc:122] Reading SavedModel debug info (if present) from: /tmp/vit_b32_fe/ +2022-09-16 09:21:53.139895: I external/org_tensorflow/tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX_VNNI FMA +To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +2022-09-16 09:21:53.241442: I external/org_tensorflow/tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:354] MLIR V1 optimization pass is not enabled +2022-09-16 09:21:53.268674: I external/org_tensorflow/tensorflow/cc/saved_model/loader.cc:228] Restoring SavedModel bundle. +2022-09-16 09:21:53.828554: I external/org_tensorflow/tensorflow/cc/saved_model/loader.cc:212] Running initialization op on SavedModel bundle at path: /tmp/vit_b32_fe/ +2022-09-16 09:21:54.105540: I external/org_tensorflow/tensorflow/cc/saved_model/loader.cc:301] SavedModel load for tags { serve }; Status: success: OK. Took 1010852 microseconds. +2022-09-16 09:21:56.018744: I external/org_tensorflow/tensorflow/compiler/xla/service/service.cc:170] XLA service 0x7f57a4014ec0 initialized for platform Host (this does not guarantee that XLA will be used). Devices: +2022-09-16 09:21:56.018777: I external/org_tensorflow/tensorflow/compiler/xla/service/service.cc:178] StreamExecutor device (0): Host, Default Version +2022-09-16 09:21:56.062819: I external/org_tensorflow/tensorflow/compiler/mlir/tensorflow/utils/dump_mlir_util.cc:263] disabling MLIR crash reproducer, set env var `MLIR_CRASH_REPRODUCER_DIRECTORY` to enable. +2022-09-16 09:22:00.695902: I external/org_tensorflow/tensorflow/compiler/jit/xla_compilation_cache.cc:478] Compiled cluster using XLA! This line is logged at most once for the lifetime of the process. +Java: Model without ConfigProto, warm up took 6.047 seconds + +BUILD SUCCESSFUL in 38s +2 actionable tasks: 1 executed, 1 up-to-date +``` From 1400a61688b4a5d6d89399052af353256b2785b8 Mon Sep 17 00:00:00 2001 From: Sebastian Lutter Date: Fri, 16 Sep 2022 18:41:25 +0200 Subject: [PATCH 04/10] Memory leak test for TF 2.9.1, java tf 0.5.0-SNAPSHOT --- java/memoryLeakRun.sh | 38 ++++++++++++++++++++++++++++++ java/src/main/java/tftest/App.java | 14 +++++++---- 2 files changed, 47 insertions(+), 5 deletions(-) create mode 100755 java/memoryLeakRun.sh diff --git a/java/memoryLeakRun.sh b/java/memoryLeakRun.sh new file mode 100755 index 0000000..034b230 --- /dev/null +++ b/java/memoryLeakRun.sh @@ -0,0 +1,38 @@ +#!/bin/bash + + +VALGRIND_OPS="--tool=memcheck --leak-check=full --smc-check=all" +cd "$(dirname "$BASH_SOURCE")" +WD=$(pwd) +# tempfile for JNI binaries +TMPDIR=$(mktemp -d) +# delete after exit using trap +trap "rm -fR $TMPDIR" EXIT + +CP_LIBS="tensorflow-core-api-0.5.0-SNAPSHOT.jar +tensorflow-core-platform-0.5.0-SNAPSHOT.jar +javacpp-1.5.7.jar +ndarray-0.4.0-SNAPSHOT.jar +protobuf-java-3.19.4.jar" + +JNI_LIBS="javacpp-1.5.7-linux-x86_64.jar +tensorflow-core-api-0.5.0-SNAPSHOT-linux-x86_64.jar" +# build jar of project (and resolve all dependencies) +./gradlew jar +# collect all dependency jars +for LIB in $CP_LIBS; do + LIBS="$(find ~/.gradle -name "${LIB}" -exec echo -n "{}" \;):${LIBS}" +done +# unpack all JNI libs to a temp location +cd $TMPDIR +for LIB in $JNI_LIBS; do + unzip -o $(find ~/.gradle -name "${LIB}" -exec echo -n "{}" \;) +done +JNIS=$(find $TMPDIR -name linux-x86_64 -type d -exec echo -n "{}:" \;) + + +cd "$WD" +echo "Run the following command:" +cd src/main/java; +echo "RUNNIND: "valgrind $VALGRIND_OPS java -cp ${LIBS}${WD}/build/libs/java.jar -Djava.library.path=${JNIS::length-1} tftest.App +valgrind $VALGRIND_OPS java -cp ${LIBS}${WD}/build/libs/java.jar -Djava.library.path=${JNIS::length-1} tftest.App diff --git a/java/src/main/java/tftest/App.java b/java/src/main/java/tftest/App.java index 89665ce..54ffa84 100644 --- a/java/src/main/java/tftest/App.java +++ b/java/src/main/java/tftest/App.java @@ -18,10 +18,14 @@ public class App { private final static String modelPath = "/tmp/vit_b32_fe/"; public static void main(String[] args) { - System.out.println("Testing query without ConfigProto"); - try (SavedModelBundle savedModel = SavedModelBundle.loader(modelPath).withTags(new String[]{"serve"}).load()) { - doInference(savedModel, "Java: Model without ConfigProto"); - } + for (int i=0;i<50;i++) { + System.out.println("Testing inference number "+i); + try (SavedModelBundle savedModel = SavedModelBundle.loader(modelPath).withTags(new String[]{"serve"}).load()) { + for (int j=0; j<50; j++) { + doInference(savedModel, "Model init "+i+" Inference "+j); + } + } + } } public static void doInference(SavedModelBundle savedModel, String msg) { @@ -31,7 +35,7 @@ public static void doInference(SavedModelBundle savedModel, String msg) { .call(Collections.singletonMap("inputs", xTensor)) .get("output_0").get()) { long end = System.currentTimeMillis(); - System.out.println(msg + ", warm up took "+((end-start)/1000f)+" seconds"); + System.out.println(msg + ", query took "+((end-start))+" ms"); } } From 67d6658fe34105978ca323efdc46b1a9bd87cc1e Mon Sep 17 00:00:00 2001 From: Sebastian Lutter Date: Fri, 16 Sep 2022 19:04:28 +0200 Subject: [PATCH 05/10] Starting with only a single run and show all leaks that are found. Guess I need to learn more valgrind --- java/memoryLeakRun.sh | 3 ++- java/src/main/java/tftest/App.java | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/java/memoryLeakRun.sh b/java/memoryLeakRun.sh index 034b230..0a13f9a 100755 --- a/java/memoryLeakRun.sh +++ b/java/memoryLeakRun.sh @@ -1,7 +1,8 @@ #!/bin/bash -VALGRIND_OPS="--tool=memcheck --leak-check=full --smc-check=all" +VALGRIND_OPS="--tool=memcheck --smc-check=all --error-limit=no " +#VALGRIND_OPS="--tool=memcheck --leak-check=full --smc-check=all" cd "$(dirname "$BASH_SOURCE")" WD=$(pwd) # tempfile for JNI binaries diff --git a/java/src/main/java/tftest/App.java b/java/src/main/java/tftest/App.java index 54ffa84..72ba53e 100644 --- a/java/src/main/java/tftest/App.java +++ b/java/src/main/java/tftest/App.java @@ -18,10 +18,10 @@ public class App { private final static String modelPath = "/tmp/vit_b32_fe/"; public static void main(String[] args) { - for (int i=0;i<50;i++) { + for (int i=0;i<1;i++) { System.out.println("Testing inference number "+i); try (SavedModelBundle savedModel = SavedModelBundle.loader(modelPath).withTags(new String[]{"serve"}).load()) { - for (int j=0; j<50; j++) { + for (int j=0; j<1; j++) { doInference(savedModel, "Model init "+i+" Inference "+j); } } From e58567f66fcb929e0a5ad550f2203964d054cb36 Mon Sep 17 00:00:00 2001 From: Sebastian Lutter Date: Fri, 16 Sep 2022 20:57:29 +0200 Subject: [PATCH 06/10] Adding valgrind flags --- java/memoryLeakRun.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/memoryLeakRun.sh b/java/memoryLeakRun.sh index 0a13f9a..f030e15 100755 --- a/java/memoryLeakRun.sh +++ b/java/memoryLeakRun.sh @@ -1,7 +1,7 @@ #!/bin/bash -VALGRIND_OPS="--tool=memcheck --smc-check=all --error-limit=no " +VALGRIND_OPS="--tool=memcheck --smc-check=all --error-limit=no --leak-check=full --track-origins=yes" #VALGRIND_OPS="--tool=memcheck --leak-check=full --smc-check=all" cd "$(dirname "$BASH_SOURCE")" WD=$(pwd) From 7867b1af743739c876910a645fa282e25e5acf53 Mon Sep 17 00:00:00 2001 From: dev server Date: Sat, 17 Sep 2022 21:35:23 +0200 Subject: [PATCH 07/10] Script to monitor memory usage over time using sar from systat package --- java/log_memory.sh | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 java/log_memory.sh diff --git a/java/log_memory.sh b/java/log_memory.sh new file mode 100644 index 0000000..0e84c28 --- /dev/null +++ b/java/log_memory.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# run sar every hour once in background (forever, kill process manually) +# sar itself creates 60 reports, every 60 seconds one report +LOGFILE="$(date +%Y_%m_%d__%H%M)_memory.log" +for ((;;)); do sar -r 60 60 >> $LOGFILE; sleep 3660; done & +# start java, load SavedModel once and do 1 million inference operations (should take about 24hours) +./gradlew run From a595cd45361c7e593c2371fd0d17d5e859596c2e Mon Sep 17 00:00:00 2001 From: dev server Date: Sat, 17 Sep 2022 21:36:17 +0200 Subject: [PATCH 08/10] Doing one model load and 1 million inference on it --- java/src/main/java/tftest/App.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/src/main/java/tftest/App.java b/java/src/main/java/tftest/App.java index 72ba53e..785fac8 100644 --- a/java/src/main/java/tftest/App.java +++ b/java/src/main/java/tftest/App.java @@ -21,7 +21,7 @@ public static void main(String[] args) { for (int i=0;i<1;i++) { System.out.println("Testing inference number "+i); try (SavedModelBundle savedModel = SavedModelBundle.loader(modelPath).withTags(new String[]{"serve"}).load()) { - for (int j=0; j<1; j++) { + for (int j=0; j<1000000; j++) { doInference(savedModel, "Model init "+i+" Inference "+j); } } From 627eebe961f8650548ade0d8103334e856be508e Mon Sep 17 00:00:00 2001 From: dev server Date: Tue, 20 Sep 2022 11:32:32 +0200 Subject: [PATCH 09/10] Run 5 million inferences. Print every 1000th line --- java/log_memory.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 java/log_memory.sh diff --git a/java/log_memory.sh b/java/log_memory.sh old mode 100644 new mode 100755 From 67efbdf2bc75c4277b476f8521e4cbeb23dec5eb Mon Sep 17 00:00:00 2001 From: dev server Date: Tue, 20 Sep 2022 11:32:55 +0200 Subject: [PATCH 10/10] Run 5 million inferences. Print every 1000th line --- java/src/main/java/tftest/App.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/java/src/main/java/tftest/App.java b/java/src/main/java/tftest/App.java index 785fac8..720e77c 100644 --- a/java/src/main/java/tftest/App.java +++ b/java/src/main/java/tftest/App.java @@ -21,21 +21,23 @@ public static void main(String[] args) { for (int i=0;i<1;i++) { System.out.println("Testing inference number "+i); try (SavedModelBundle savedModel = SavedModelBundle.loader(modelPath).withTags(new String[]{"serve"}).load()) { - for (int j=0; j<1000000; j++) { - doInference(savedModel, "Model init "+i+" Inference "+j); + for (int j=0; j<5000000; j++) { + doInference(savedModel, "Model init "+i+" Inference "+j, j%1000==0); } } } } - public static void doInference(SavedModelBundle savedModel, String msg) { + public static void doInference(SavedModelBundle savedModel, String msg, boolean printOut) { long start = System.currentTimeMillis(); try (TFloat32 xTensor = TFloat32.tensorOf(NdArrays.ofFloats(Shape.of(1,244,244,3))); TFloat32 zTensor = (TFloat32) savedModel .call(Collections.singletonMap("inputs", xTensor)) .get("output_0").get()) { long end = System.currentTimeMillis(); - System.out.println(msg + ", query took "+((end-start))+" ms"); + if (printOut) { + System.out.println(msg + ", query took "+((end-start))+" ms"); + } } }