Test accuracy

.gitignore

@@ -100,3 +100,5 @@ simulated_results
 
 # Evaluation files
 src/server/evaluations/evaluations.csv
+datasets/
+sciot-fomo-models-export/

README_BENCHMARK.md

@@ -0,0 +1,40 @@
+# SCIoT - Istruzioni di Avvio Benchmark
+
+
+
+Per avviare il server in ascolto e inizializzare il modello per l'offloading, eseguire dalla root del progetto, evita il prossimo comando se vuoi testare tutto in locale
+
+```bash
+python3 src/server/edge/run_edge.py
+```
+
+Se vuoi eseguire il benchmark sul pc assicurati che in src/client/python/benchmark.py alla riga 15 leggi
+
+```yaml
+CLIENT_SCRIPT = BASE_DIR / "http_clientCAMlaptopBench.py"
+```
+Se lo esegui su su un raspberry allora dovrai avere
+
+```yaml
+CLIENT_SCRIPT = BASE_DIR / "http_clientCAMpiBench.py"
+```
+
+
+assicurati che nel file src/client/python/http_config.yaml vedi
+```yaml
+debug_cprofiler: false
+```
+ora, dalla root del progetto, puoi eseguire il benchmark con
+
+```bash
+python3 src/client/python/benchmark.py
+```
+una volta terminato troverai i file json nella cartella "logs", lancia il seguente comando per generare grafici
+
+```bash
+python3 plot_results.py
+```
+
+
+
+

analisi test/cprofiler multi thread/client_stats_pippo_96.json

@@ -0,0 +1,61 @@
+{
+    "hardware": "pippo_96",
+    "phases": {
+        "client_image_acquisition": {
+            "count": 30,
+            "wall_time_ms": {
+                "mean": 0.25,
+                "p95": 0.35
+            },
+            "bottleneck_analysis": {
+                "cpu_time_ms_mean": 0.25,
+                "io_wait_ms_mean": 0.0
+            },
+            "memory_analysis": {
+                "mem_used_kb_max": 0.81
+            }
+        },
+        "client_ask_offloading": {
+            "count": 30,
+            "wall_time_ms": {
+                "mean": 90.06,
+                "p95": 125.79
+            },
+            "bottleneck_analysis": {
+                "cpu_time_ms_mean": 55.42,
+                "io_wait_ms_mean": 34.64
+            },
+            "memory_analysis": {
+                "mem_used_kb_max": 153.47
+            }
+        },
+        "client_local_inference": {
+            "count": 30,
+            "wall_time_ms": {
+                "mean": 2.13,
+                "p95": 1.51
+            },
+            "bottleneck_analysis": {
+                "cpu_time_ms_mean": 2.14,
+                "io_wait_ms_mean": 0.0
+            },
+            "memory_analysis": {
+                "mem_used_kb_max": 438.94
+            }
+        },
+        "client_network_upload": {
+            "count": 29,
+            "wall_time_ms": {
+                "mean": 306.03,
+                "p95": 424.51
+            },
+            "bottleneck_analysis": {
+                "cpu_time_ms_mean": 28.23,
+                "io_wait_ms_mean": 277.8
+            },
+            "memory_analysis": {
+                "mem_used_kb_max": 1.54
+            }
+        }
+    }
+}

analisi test/cprofiler multi thread/server_deep_analysis_Edge_Server_20260428_173148.txt

@@ -0,0 +1,38 @@
+         684021 function calls (668894 primitive calls) in 26.429 seconds
+
+   Ordered by: cumulative time
+   List reduced from 1593 to 30 due to restriction <30>
+
+   ncalls  tottime  percall  cumtime  percall filename:lineno(function)
+        1    0.000    0.000   26.329   26.329 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/src/server/communication/http_server.py:226(run)
+     7141    0.225    0.000   26.283    0.004 /usr/lib/python3.11/asyncio/base_events.py:1845(_run_once)
+        1    0.000    0.000   26.279   26.279 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/uvicorn/main.py:464(run)
+        1    0.000    0.000   26.277   26.277 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/uvicorn/server.py:64(run)
+        1    0.000    0.000   26.276   26.276 /usr/lib/python3.11/asyncio/runners.py:160(run)
+        1    0.000    0.000   26.275   26.275 /usr/lib/python3.11/asyncio/runners.py:86(run)
+        1    0.000    0.000   26.275   26.275 /usr/lib/python3.11/asyncio/base_events.py:617(run_until_complete)
+        1    0.023    0.023   26.275   26.275 /usr/lib/python3.11/asyncio/base_events.py:593(run_forever)
+     7141    0.128    0.000   23.437    0.003 /usr/lib/python3.11/selectors.py:451(select)
+     7141   23.270    0.003   23.270    0.003 {method 'poll' of 'select.epoll' objects}
+     9206    0.065    0.000    2.511    0.000 /usr/lib/python3.11/asyncio/events.py:78(_run)
+     9206    0.046    0.000    2.446    0.000 {method 'run' of '_contextvars.Context' objects}
+     4163    0.023    0.000    1.450    0.000 /usr/lib/python3.11/asyncio/selector_events.py:951(_read_ready)
+     4163    0.036    0.000    1.428    0.000 /usr/lib/python3.11/asyncio/selector_events.py:991(_read_ready__data_received)
+     4163    0.042    0.000    1.234    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/uvicorn/protocols/http/h11_impl.py:168(data_received)
+     4314    0.132    0.000    1.155    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/uvicorn/protocols/http/h11_impl.py:174(handle_events)
+     4213    0.008    0.000    0.815    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/uvicorn/protocols/http/h11_impl.py:401(run_asgi)
+     4214    0.008    0.000    0.807    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/uvicorn/middleware/proxy_headers.py:27(__call__)
+     8630    0.116    0.000    0.789    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/h11/_connection.py:438(next_event)
+     4214    0.006    0.000    0.781    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/fastapi/applications.py:1051(__call__)
+     4214    0.006    0.000    0.775    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/starlette/applications.py:108(__call__)
+     4214    0.005    0.000    0.769    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/starlette/middleware/errors.py:150(__call__)
+     4214    0.008    0.000    0.764    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/starlette/middleware/exceptions.py:46(__call__)
+8426/4213    0.010    0.000    0.752    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/starlette/_exception_handler.py:31(wrapped_app)
+     4214    0.005    0.000    0.747    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/starlette/routing.py:710(__call__)
+     4214    0.014    0.000    0.742    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/starlette/routing.py:716(app)
+     4213    0.006    0.000    0.704    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/starlette/routing.py:279(handle)
+     4213    0.008    0.000    0.698    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/starlette/routing.py:69(app)
+     4213    0.007    0.000    0.683    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/starlette/routing.py:72(app)
+     4213    0.018    0.000    0.504    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/fastapi/routing.py:240(app)
+
+

analisi test/cprofiler multi thread/server_stats.json

@@ -0,0 +1,61 @@
+{
+    "hardware": "Edge_Server",
+    "phases": {
+        "server_payload_decoding": {
+            "count": 50,
+            "wall_time_ms": {
+                "mean": 0.42,
+                "p95": 0.53
+            },
+            "bottleneck_analysis": {
+                "cpu_time_ms_mean": 0.42,
+                "io_wait_ms_mean": 0.0
+            },
+            "memory_analysis": {
+                "mem_used_kb_max": 0.0
+            }
+        },
+        "server_state_update": {
+            "count": 99,
+            "wall_time_ms": {
+                "mean": 17.53,
+                "p95": 39.56
+            },
+            "bottleneck_analysis": {
+                "cpu_time_ms_mean": 17.53,
+                "io_wait_ms_mean": 0.0
+            },
+            "memory_analysis": {
+                "mem_used_kb_max": 0.0
+            }
+        },
+        "server_algo_and_io": {
+            "count": 50,
+            "wall_time_ms": {
+                "mean": 1.33,
+                "p95": 1.0
+            },
+            "bottleneck_analysis": {
+                "cpu_time_ms_mean": 1.37,
+                "io_wait_ms_mean": 0.0
+            },
+            "memory_analysis": {
+                "mem_used_kb_max": 107.41
+            }
+        },
+        "server_inference_execution": {
+            "count": 49,
+            "wall_time_ms": {
+                "mean": 101.1,
+                "p95": 103.16
+            },
+            "bottleneck_analysis": {
+                "cpu_time_ms_mean": 101.15,
+                "io_wait_ms_mean": 0.0
+            },
+            "memory_analysis": {
+                "mem_used_kb_max": 6367.9
+            }
+        }
+    }
+}

analisi test/cprofiler single thread/client_stats_pippo_96.json

@@ -0,0 +1,61 @@
+{
+    "hardware": "pippo_96",
+    "phases": {
+        "client_image_acquisition": {
+            "count": 30,
+            "wall_time_ms": {
+                "mean": 0.21,
+                "p95": 0.36
+            },
+            "bottleneck_analysis": {
+                "cpu_time_ms_mean": 0.21,
+                "io_wait_ms_mean": 0.01
+            },
+            "memory_analysis": {
+                "mem_used_kb_max": 0.0
+            }
+        },
+        "client_ask_offloading": {
+            "count": 30,
+            "wall_time_ms": {
+                "mean": 80.18,
+                "p95": 110.63
+            },
+            "bottleneck_analysis": {
+                "cpu_time_ms_mean": 48.0,
+                "io_wait_ms_mean": 32.18
+            },
+            "memory_analysis": {
+                "mem_used_kb_max": 153.14
+            }
+        },
+        "client_local_inference": {
+            "count": 30,
+            "wall_time_ms": {
+                "mean": 2.02,
+                "p95": 1.46
+            },
+            "bottleneck_analysis": {
+                "cpu_time_ms_mean": 2.03,
+                "io_wait_ms_mean": 0.0
+            },
+            "memory_analysis": {
+                "mem_used_kb_max": 438.92
+            }
+        },
+        "client_network_upload": {
+            "count": 29,
+            "wall_time_ms": {
+                "mean": 384.87,
+                "p95": 1000.19
+            },
+            "bottleneck_analysis": {
+                "cpu_time_ms_mean": 24.29,
+                "io_wait_ms_mean": 360.57
+            },
+            "memory_analysis": {
+                "mem_used_kb_max": 1.54
+            }
+        }
+    }
+}

analisi test/cprofiler single thread/server_deep_analysis_Edge_Server_20260428_174624.txt

@@ -0,0 +1,38 @@
+         642737 function calls (628643 primitive calls) in 29.740 seconds
+
+   Ordered by: cumulative time
+   List reduced from 1593 to 30 due to restriction <30>
+
+   ncalls  tottime  percall  cumtime  percall filename:lineno(function)
+        1    0.000    0.000   29.700   29.700 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/src/server/communication/http_server.py:226(run)
+        1    0.000    0.000   29.658   29.658 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/uvicorn/main.py:464(run)
+        1    0.000    0.000   29.656   29.656 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/uvicorn/server.py:64(run)
+        1    0.000    0.000   29.655   29.655 /usr/lib/python3.11/asyncio/runners.py:160(run)
+        1    0.000    0.000   29.654   29.654 /usr/lib/python3.11/asyncio/runners.py:86(run)
+        1    0.000    0.000   29.654   29.654 /usr/lib/python3.11/asyncio/base_events.py:617(run_until_complete)
+        1    0.022    0.022   29.654   29.654 /usr/lib/python3.11/asyncio/base_events.py:593(run_forever)
+     6472    0.224    0.000   29.632    0.005 /usr/lib/python3.11/asyncio/base_events.py:1845(_run_once)
+     6472    0.124    0.000   26.840    0.004 /usr/lib/python3.11/selectors.py:451(select)
+     6472   26.678    0.004   26.678    0.004 {method 'poll' of 'select.epoll' objects}
+     8583    0.063    0.000    2.465    0.000 /usr/lib/python3.11/asyncio/events.py:78(_run)
+     8583    0.045    0.000    2.401    0.000 {method 'run' of '_contextvars.Context' objects}
+     3821    0.022    0.000    1.413    0.000 /usr/lib/python3.11/asyncio/selector_events.py:951(_read_ready)
+     3821    0.036    0.000    1.391    0.000 /usr/lib/python3.11/asyncio/selector_events.py:991(_read_ready__data_received)
+     3821    0.042    0.000    1.200    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/uvicorn/protocols/http/h11_impl.py:168(data_received)
+     3971    0.125    0.000    1.120    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/uvicorn/protocols/http/h11_impl.py:174(handle_events)
+     3870    0.008    0.000    0.797    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/uvicorn/protocols/http/h11_impl.py:401(run_asgi)
+     3871    0.007    0.000    0.790    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/uvicorn/middleware/proxy_headers.py:27(__call__)
+     7947    0.112    0.000    0.769    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/h11/_connection.py:438(next_event)
+     3871    0.005    0.000    0.765    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/fastapi/applications.py:1051(__call__)
+     3871    0.006    0.000    0.760    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/starlette/applications.py:108(__call__)
+     3871    0.005    0.000    0.753    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/starlette/middleware/errors.py:150(__call__)
+     3871    0.008    0.000    0.748    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/starlette/middleware/exceptions.py:46(__call__)
+7740/3870    0.010    0.000    0.737    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/starlette/_exception_handler.py:31(wrapped_app)
+     3871    0.005    0.000    0.732    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/starlette/routing.py:710(__call__)
+     3871    0.013    0.000    0.727    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/starlette/routing.py:716(app)
+     3870    0.006    0.000    0.692    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/starlette/routing.py:279(handle)
+     3870    0.008    0.000    0.686    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/starlette/routing.py:69(app)
+     3870    0.007    0.000    0.671    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/starlette/routing.py:72(app)
+     3870    0.017    0.000    0.502    0.000 /home/matteo/Desktop/New_SCIoT/SCIoT_python_client/.venv/lib/python3.11/site-packages/fastapi/routing.py:240(app)
+
+

analisi test/cprofiler single thread/server_stats.json

@@ -0,0 +1,61 @@
+{
+    "hardware": "Edge_Server",
+    "phases": {
+        "server_payload_decoding": {
+            "count": 50,
+            "wall_time_ms": {
+                "mean": 0.44,
+                "p95": 0.52
+            },
+            "bottleneck_analysis": {
+                "cpu_time_ms_mean": 0.44,
+                "io_wait_ms_mean": 0.0
+            },
+            "memory_analysis": {
+                "mem_used_kb_max": 0.0
+            }
+        },
+        "server_state_update": {
+            "count": 99,
+            "wall_time_ms": {
+                "mean": 17.77,
+                "p95": 40.95
+            },
+            "bottleneck_analysis": {
+                "cpu_time_ms_mean": 17.77,
+                "io_wait_ms_mean": 0.0
+            },
+            "memory_analysis": {
+                "mem_used_kb_max": 0.0
+            }
+        },
+        "server_algo_and_io": {
+            "count": 50,
+            "wall_time_ms": {
+                "mean": 1.01,
+                "p95": 1.03
+            },
+            "bottleneck_analysis": {
+                "cpu_time_ms_mean": 1.05,
+                "io_wait_ms_mean": 0.0
+            },
+            "memory_analysis": {
+                "mem_used_kb_max": 0.0
+            }
+        },
+        "server_inference_execution": {
+            "count": 49,
+            "wall_time_ms": {
+                "mean": 99.21,
+                "p95": 100.26
+            },
+            "bottleneck_analysis": {
+                "cpu_time_ms_mean": 99.25,
+                "io_wait_ms_mean": 0.0
+            },
+            "memory_analysis": {
+                "mem_used_kb_max": 6366.91
+            }
+        }
+    }
+}