Generate benchmarks automatically

.github/workflows/documentation.yml

@@ -7,19 +7,118 @@ on:
   push:
     branches:
       - main
+  pull_request:
+
 permissions:
   contents: write
 jobs:
+  build_firmware_size:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Cache pip
+      uses: actions/cache@v2
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Cache PlatformIO
+      uses: actions/cache@v2
+      with:
+        path: ~/.platformio
+        key: ${{ runner.os }}-${{ hashFiles('**/lockfiles') }}
+    - name: Set up Python
+      uses: actions/setup-python@v4
+    - name: Install PlatformIO
+      run: |
+        python -m pip install --upgrade pip
+        pip install --upgrade platformio
+    - name: Run PlatformIO
+      run: pio ci --lib="." --build-dir="${{ github.workspace }}/../build" --keep-build-dir --project-conf="./tests/benchmarks/firmware_size/platformio.ini" ./tests/benchmarks/firmware_size/main.cpp
+    - name: Move firmware files # change path to location without parent dir ('..') statement (to make upload-artifact happy)
+      run: |
+        mkdir firmware
+        mv "${{ github.workspace }}/../build/.pio/build/v16/firmware.elf"  firmware/firmware_v16.elf
+        mv "${{ github.workspace }}/../build/.pio/build/v201/firmware.elf" firmware/firmware_v201.elf
+    - name: Upload firmware linker files
+      uses: actions/upload-artifact@v4
+      with:
+        name: Firmware linker files
+        path: firmware
+        if-no-files-found: error
+        retention-days: 1
+
+  evaluate_firmware:
+    needs: build_firmware_size
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
+      with:
+        python-version: 3.x
+    - uses: actions/cache@v2
+      with:
+        key: ${{ github.ref }}
+        path: .cache
+    - name: Install Python dependencies
+      run: pip install pandas 
+    - name: Get build tools
+      run: |
+        sudo apt update
+        sudo apt install build-essential cmake ninja-build
+        sudo apt -y install gcc-9 g++-9
+        g++ --version
+    - name: Check out bloaty
+      uses: actions/checkout@v3
+      with:
+        repository: google/bloaty
+        ref: 379d5305670c00c36a57e608079fd253f13bde63
+        path: tools/bloaty
+        submodules: recursive
+    - name: Install bloaty
+      run: |
+        cmake -B tools/bloaty/build -G Ninja -S tools/bloaty
+        cmake --build tools/bloaty/build -j 32
+    - name: Get firmware linker files
+      uses: actions/download-artifact@v4
+      with:
+        name: Firmware linker files
+        path: firmware
+    - name: Run bloaty
+      run: |
+        mkdir docs/assets
+        mkdir docs/assets/tables
+        tools/bloaty/build/bloaty firmware/firmware_v16.elf  -d compileunits --csv -n 0 > docs/assets/tables/bloaty_v16.csv
+        tools/bloaty/build/bloaty firmware/firmware_v201.elf -d compileunits --csv -n 0 > docs/assets/tables/bloaty_v201.csv
+    - name: Evaluate and create reports
+      run: python tests/benchmarks/scripts/eval_firmware_size.py
+    - name: Upload reports
+      uses: actions/upload-artifact@v4
+      with:
+        name: Firmware size reports CSV
+        path: docs/assets/tables
+        if-no-files-found: error
+
   deploy:
+    needs: evaluate_firmware
     runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/main'
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: 3.x
-      - uses: actions/cache@v2
-        with:
-          key: ${{ github.ref }}
-          path: .cache
-      - run: pip install mkdocs-material 
-      - run: mkdocs gh-deploy --force
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
+      with:
+        python-version: 3.x
+    - uses: actions/cache@v2
+      with:
+        key: ${{ github.ref }}
+        path: .cache
+    - name: Install Python dependencies
+      run: pip install pandas mkdocs-material mkdocs-table-reader-plugin
+    - name: Get firmware size reports
+      uses: actions/download-artifact@v4
+      with:
+        name: Firmware size reports CSV
+        path: docs/assets/tables
+    - name: Run mkdocs
+      run: mkdocs gh-deploy --force

.gitignore

@@ -6,3 +6,4 @@ src/ArduinoJson*
 src/main.cpp
 tests/helpers/ArduinoJson*
 coverage.info
+docs/assets

CHANGELOG.md

@@ -22,6 +22,7 @@
 - Support for TransactionMessageAttempts/-RetryInterval ([#345](https://github.com/matth-x/MicroOcpp/pull/345))
 - Heap profiler and custom allocator support ([#350](https://github.com/matth-x/MicroOcpp/pull/350))
 - Migration of persistent storage ([#355](https://github.com/matth-x/MicroOcpp/pull/355))
+- Benchmarks pipeline ([#369](https://github.com/matth-x/MicroOcpp/pull/369))
 
 ### Removed
 

docs/benchmarks.md

@@ -0,0 +1,47 @@
+# Benchmarks
+
+Microcontrollers have tight hardware constraints which affect how much resources the firmware can demand. It is important to make sure that the available resources are not depleted to allow for robust operation and that there is sufficient flash head room to allow for future software upgrades.
+
+In general, microcontrollers have three relevant hardware constraints:
+
+- Limited processing speed
+- Limited memory size
+- Limited flash size
+
+For OCPP, the relevant bottlenecks are especially the memory and flash size. The processing speed is no concern, since OCPP is not computationally complex and does not include any extensive planning algorithms on the charger size. A previous [benchmark on the ESP-IDF](https://github.com/matth-x/MicroOcpp-benchmark) showed that the processing times are in the lower milliseconds range and are probably outweighed by IO times and network round trip times.
+
+However, the memory and flash requirements are important figures, because the device model of OCPP has a significant size. The microcontroller needs to keep the model data in the heap memory for the largest part and the firmware which covers the corresponding processing routines needs to have sufficient space on flash.
+
+This chapter presents benchmarks of the memory and flash requirements. They should help to determine the required microcontroller capabilities, or to give general insights for taking further action on optimizing the firmware.
+
+## Firmware size
+
+When compiling a firmware with MicroOCPP, the resulting binary will contain functionality which is not related to OCPP, like hardware drivers, modules which are shared, like MbedTLS and the actual MicroOCPP object files. The size of the latter is the final flash requirement of MicroOCPP.
+
+For the flash benchmark, the profiler compiles a [dummy OCPP firmware](https://github.com/matth-x/MicroOcpp/tree/main/tests/benchmarks/firmware_size/main.cpp), analyzes the size of the compilation units using [bloaty](https://github.com/google/bloaty) and evaluates the bloaty report using a [Python script](https://github.com/matth-x/MicroOcpp/tree/main/tests/benchmarks/scripts/eval_firmware_size.py). To give realistic results, the firwmare is compiled with `-Os`, no RTTI or exceptions and newlib as the standard C library. The following tables show the results.
+
+### OCPP 1.6
+
+The following table shows the cumulated size of the objects files per module. The Module category consists of the OCPP 2.0.1 functional blocks, OCPP 1.6 feature profiles and general functionality which is shared accross the library. If a feature of the implementation falls under both an OCPP 2.0.1 functional block and OCPP 1.6 feature profile definition, it is preferrably assigned to the OCPP 2.0.1 category. This allows for better comparability between both OCPP versions.
+
+**Table 1: Firmware size per Module**
+
+{{ read_csv('modules_v16.csv') }}
+
+### OCPP 2.0.1
+
+**Table 2: Firmware size per Module**
+
+{{ read_csv('modules_v201.csv') }}
+
+## Full data sets
+
+This section contains the raw data which is the basis for the evaluations above.
+
+**Table 3: All compilation units for OCPP 1.6 firmware**
+
+{{ read_csv('compile_units_v16.csv') }}
+
+**Table 4: All compilation units for OCPP 2.0.1 firmware**
+
+{{ read_csv('compile_units_v201.csv') }}

mkdocs.yml

@@ -48,3 +48,8 @@ theme:
 
 extra_css:
   - stylesheets/extra.css
+
+plugins:
+  - search
+  - table-reader:
+      data_path: "docs/assets/tables"

tests/benchmarks/firmware_size/main.cpp

@@ -0,0 +1,68 @@
+// matth-x/MicroOcpp
+// Copyright Matthias Akstaller 2019 - 2024
+// MIT License
+
+#include <MicroOcpp.h>
+#include <MicroOcpp_c.h>
+#include <MicroOcpp/Model/FirmwareManagement/FirmwareService.h>
+#include <MicroOcpp/Model/Diagnostics/DiagnosticsService.h>
+
+MicroOcpp::LoopbackConnection g_loopback;
+
+void setup() {
+
+    ocpp_deinitialize();
+
+#if MO_ENABLE_V201
+    mocpp_initialize(g_loopback, ChargerCredentials::v201(),MicroOcpp::makeDefaultFilesystemAdapter(MicroOcpp::FilesystemOpt::Use_Mount_FormatOnFail),true,MicroOcpp::ProtocolVersion(2,0,1));
+#else
+    mocpp_initialize(g_loopback, ChargerCredentials());
+#endif
+
+    ocpp_beginTransaction("");
+    ocpp_beginTransaction_authorized("","");
+    ocpp_endTransaction("","");
+    ocpp_endTransaction_authorized("","");
+    ocpp_isTransactionActive();
+    ocpp_isTransactionRunning();
+    ocpp_getTransactionIdTag();
+    ocpp_getTransaction();
+    ocpp_ocppPermitsCharge();
+    ocpp_getChargePointStatus();
+    ocpp_setConnectorPluggedInput([] () {return false;});
+    ocpp_setEnergyMeterInput([] () {return 0;});
+    ocpp_setPowerMeterInput([] () {return 0.f;});
+    ocpp_setSmartChargingPowerOutput([] (float) {});
+    ocpp_setSmartChargingCurrentOutput([] (float) {});
+    ocpp_setSmartChargingOutput([] (float,float,int) {});
+    ocpp_setEvReadyInput([] () {return false;});
+    ocpp_setEvseReadyInput([] () {return false;});
+    ocpp_addErrorCodeInput([] () {return (const char*)nullptr;});
+    addErrorDataInput([] () {return MicroOcpp::ErrorData("");});
+    ocpp_addMeterValueInputFloat([] () {return 0.f;},"","","","");
+    ocpp_setOccupiedInput([] () {return false;});
+    ocpp_setStartTxReadyInput([] () {return false;});
+    ocpp_setStopTxReadyInput([] () {return false;});
+    ocpp_setTxNotificationOutput([] (OCPP_Transaction*, OCPP_TxNotification) {});
+
+#if MO_ENABLE_CONNECTOR_LOCK
+    ocpp_setOnUnlockConnectorInOut([] () {return UnlockConnectorResult_UnlockFailed;});
+#endif
+
+    isOperative();
+    setOnResetNotify([] (bool) {return false;});
+    setOnResetExecute([] (bool) {return false;});
+    getFirmwareService()->getFirmwareStatus();
+    getDiagnosticsService()->getDiagnosticsStatus();
+
+#if MO_ENABLE_CERT_MGMT
+    setCertificateStore(nullptr);
+#endif
+
+    getOcppContext();
+
+}
+
+void loop() {
+    mocpp_loop();
+}

tests/benchmarks/firmware_size/platformio.ini

@@ -0,0 +1,38 @@
+; matth-x/MicroOcpp
+; Copyright Matthias Akstaller 2019 - 2024
+; MIT License
+
+[common]
+platform = espressif32@6.8.1
+board = esp-wrover-kit
+framework = arduino
+lib_deps =
+    bblanchon/ArduinoJson@6.20.1
+build_flags=
+    -D MO_DBG_LEVEL=MO_DL_NONE ; don't take debug messages into account
+    -D MO_CUSTOM_WS
+
+[env:v16]
+platform = ${common.platform}
+board = ${common.board}
+framework = ${common.framework}
+lib_deps = ${common.lib_deps}
+build_flags =
+    ${common.build_flags}
+    -D MO_ENABLE_MBEDTLS=1
+    -D MO_ENABLE_CERT_MGMT=1
+    -D MO_ENABLE_RESERVATION=1
+    -D MO_ENABLE_LOCAL_AUTH=1
+    -D MO_REPORT_NOERROR=1
+    -D MO_ENABLE_CONNECTOR_LOCK=1
+
+[env:v201]
+platform = ${common.platform}
+board = ${common.board}
+framework = ${common.framework}
+lib_deps = ${common.lib_deps}
+build_flags =
+    ${common.build_flags}
+    -D MO_ENABLE_V201=1
+    -D MO_ENABLE_MBEDTLS=1
+    -D MO_ENABLE_CERT_MGMT=1