embedded/FreeRTOS
1
0
Fork 0
mirror of https://github.com/FreeRTOS/FreeRTOS.git synced 2026-07-30 01:33:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add support for generating JSON report (#81)

Browse Source 
Add support for generating JSON report

Passing -g option will generate a JSON file containing size information
about all the libraries.

It also adds the support to patch source code before calculating size and
revert the patch afterwards. This should ideally be not needed as we should
be able to control all the configurations via config files. OTA currently defines
the logging level in a header file as opposed to taking it from a config file.
As a result, we need to patch the header file before calculating the size as
we want to turn off logging while calculating size. Later when logging
level is moved to a config file, it will not be needed and we will turn
off logging in the header file contained with the memory estimator tool.

Signed-off-by: Gaurav Aggarwal <aggarg@amazon.com>
This commit is contained in:
Gaurav-Aggarwal-AWS
2020-06-05 13:10:51 -07:00
committed by GitHub
parent 91068f0862
commit 2acb08be81
5 changed files with 252 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tools/memory_estimator/README.md
View File

@@ -20,8 +20,8 @@ Where:
| ---------- | --------- | ------- | ---------- |
| -p | --lts-path | | Path to the directory containing FreeRTOS LTS code. |
| -o | --optimization | O0 | Compiler optimization level (O0, Os etc). |
| -l | --lib | `mqtt` | The library to calculate the memory estimates for. Currently supported libraries are: `mqtt`, `light-mqtt`, `https`, `shadow`, `jobs`, `ota`, `kernel`|
| -l | --lib | `mqtt` | The library to calculate the memory estimates for. Currently supported libraries are: `mqtt`, `light-mqtt`, `https`, `shadow`, `jobs`, `ota-mqtt`, `ota-http`, `kernel`|
| -c | --compiler | arm-none-eabi-gcc | Compiler to use. |
| -s | --sizetool | arm-none-eabi-size | Size tool to use. |
| -d | --dontclean | | The generated artifacts, which include the generated Makefile and built object files, are deleted by default. Pass `-d` to ensure that the generated artifacts are not deleted. |
| -g | --generate-report | | Pass `-g` to generate a JSON report containing sizes for all libraries. If this option is used, values for `--optimization`, `--lib` and `--dontclean` are ignored. |

75
tools/memory_estimator/json_report_generator.py Normal file
View File

@@ -0,0 +1,75 @@
import os
import copy
from operator import truediv
# Some of the file names are customized when diplaying on the website. This dict
# contains the mappings of original file name to customized name with additional
# information.
__FILE_NAME_TO_DISPLAY_NAME__ = {
"http_parser.c": "http_parser.c (third-party utility)"
}
def convert_size_to_kb(byte_size):
kb_size = round(truediv(int(byte_size),1024),1)
if (kb_size == 0.0):
return 0.1
else:
return kb_size
def parse_sizes(sizes):
'''
It assumes the Berkley output format:
text data bss dec hex filename
Returns a dict of the following format:
{
'file_name':'size',
'file_name':'size'
}
'''
filename_to_size_dict = {}
# The first line is the size tool command and the second line is the
# header. Therefore, we start from third line which contains actual sizes.
for size in sizes[2:]:
parts = size.split()
filename = os.path.basename(parts[5])
c_filename = str(filename.replace('.o','.c').strip())
text_size = parts[0].strip()
text_size_in_kb = convert_size_to_kb(text_size)
filename_to_size_dict[c_filename] = text_size_in_kb
return filename_to_size_dict
def update_json_report(lib_name, o1_sizes, os_sizes, size_description, lib_report_template, generated_json_report):
filename_to_o1_size_dict = parse_sizes(o1_sizes)
filename_to_os_size_dict = parse_sizes(os_sizes)
# Make a copy of the template.
lib_report = copy.deepcopy(lib_report_template)
# Populate files in the library report.
for filename in filename_to_o1_size_dict:
file_report = {}
file_report['file_name'] = __FILE_NAME_TO_DISPLAY_NAME__.get(filename, filename)
file_report['o1_size'] = '{:.1f}K'.format(filename_to_o1_size_dict[filename])
file_report['os_size']= '{:.1f}K'.format(filename_to_os_size_dict[filename])
lib_report['files'].append(file_report)
# Update total size of the library.
lib_report['total']['total_o1'] = '{:.1f}K'.format(sum(filename_to_o1_size_dict.values()))
lib_report['total']['total_os'] = '{:.1f}K'.format(sum(filename_to_os_size_dict.values()))
# Add the size description.
lib_report['table_header'] = size_description
# Add the library report to the report.
generated_json_report[lib_name] = lib_report

145
tools/memory_estimator/memory_estimator.py
View File

@@ -3,8 +3,10 @@
import io
import os
import re
import json
import argparse
import subprocess
from json_report_generator import update_json_report
from makefile_generator import generate_makefile_from_template
@@ -12,11 +14,14 @@ __THIS_FILE_PATH__ = os.path.dirname(os.path.abspath(__file__))
__MAKE_FILE_TEMPLATE__ = os.path.join(__THIS_FILE_PATH__, 'template', 'Makefile.template')
__GENERATED_MAKE_FILE__ = os.path.join(__THIS_FILE_PATH__, 'Makefile')
__JSON_REPORT_TEMPLATE__ = os.path.join(__THIS_FILE_PATH__, 'template', 'report.json.template')
__GENERATED_JSON_REPORT__ = os.path.join(__THIS_FILE_PATH__, 'freertos_lts_memory_estimates.json')
__REPORT_LIBS_JSON__ = os.path.join(__THIS_FILE_PATH__, 'template', 'report_libs.json')
__FREERTOS_SRC_DIR__ = os.path.join('FreeRTOS', 'Source')
__FREERTOS_PLUS_SRC_DIR__ = os.path.join('FreeRTOS-Plus', 'Source')
__IOT_LIBS_DIR__ = os.path.join(__FREERTOS_PLUS_SRC_DIR__, 'FreeRTOS-IoT-Libraries')
__LIB_NAME_TO_SRC_DIRS_MAPPING__ = {
'light-mqtt' : [
os.path.join(__IOT_LIBS_DIR__, 'c_sdk', 'standard', 'mqtt', 'src', 'iot_mqtt_lightweight_api.c'),
@@ -37,7 +42,13 @@ __LIB_NAME_TO_SRC_DIRS_MAPPING__ = {
os.path.join(__IOT_LIBS_DIR__, 'c_sdk', 'aws', 'jobs', 'src'),
os.path.join(__IOT_LIBS_DIR__, 'c_sdk', 'aws', 'common', 'src')
],
'ota' : [
'ota-mqtt' : [
os.path.join(__IOT_LIBS_DIR__, 'c_sdk', 'aws', 'ota', 'src', 'aws_iot_ota_agent.c'),
os.path.join(__IOT_LIBS_DIR__, 'c_sdk', 'aws', 'ota', 'src', 'aws_iot_ota_interface.c'),
os.path.join(__IOT_LIBS_DIR__, 'c_sdk', 'aws', 'ota', 'src', 'mqtt', 'aws_iot_ota_mqtt.c'),
os.path.join(__IOT_LIBS_DIR__, 'c_sdk', 'aws', 'ota', 'src', 'mqtt', 'aws_iot_ota_cbor.c')
],
'ota-http' : [
os.path.join(__IOT_LIBS_DIR__, 'c_sdk', 'aws', 'ota', 'src')
],
'kernel' : [
@@ -52,6 +63,70 @@ __LIB_NAME_TO_SRC_DIRS_MAPPING__ = {
}
def apply_patches(freertos_lts, lib_name):
patches = {}
# This should ideally be empty as we should be able to control all the
# configurations via config files. However, OTA currently defines the
# logging level in a header file as opposed to taking it from a config file.
# As a result, we need to patch the header file before calculating the size
# as we want to turn off logging while calculating size. Later when logging
# level is moved to a config file, it will not be needed and we will turn
# off logging in the header file tools\memory_estimator\config_files\aws_ota_agent_config.h.
if 'ota' in lib_name:
ota_agent_header_file = os.path.join(freertos_lts, __IOT_LIBS_DIR__, 'c_sdk', 'aws', 'ota', 'include', 'aws_iot_ota_agent.h')
with open(ota_agent_header_file) as ota_agent_header_file_handle:
original_content = ota_agent_header_file_handle.read()
# Turn off logging in the OTA code while calculating sizes.
patched_content = original_content.replace('#define OTA_DEBUG_LOG_LEVEL 1', '#define OTA_DEBUG_LOG_LEVEL 0')
with open(ota_agent_header_file, 'w') as ota_agent_header_file_handle:
ota_agent_header_file_handle.write(patched_content)
patches[ota_agent_header_file] = original_content
return patches
def revert_patches(patches):
for patched_file in patches:
original_content = patches[patched_file]
with open(patched_file, 'w') as patched_file_handle:
patched_file_handle.write(original_content)
def calculate_sizes(freertos_lts, optimization, lib_name, compiler, sizetool, dontclean):
# Apply the required patches to the source code.
patches = apply_patches(freertos_lts, lib_name)
# Generate Makefile.
generate_makefile(freertos_lts, optimization, lib_name)
# Run make build.
warnings = make(compiler)
# Run make size.
calculated_sizes = calculate_size(sizetool)
# Remove the generated artifacts.
if not dontclean:
clean()
remove_generated_makefile()
# Print the generated warnings.
pretty_print_warnings(warnings)
# Print the calculated sizes.
pretty_print_sizes(calculated_sizes)
# Revert the applied patches to the source code.
revert_patches(patches)
return calculated_sizes
def generate_makefile(freertos_lts, optimization, lib_name):
freertos_src_dir = os.path.join(freertos_lts, __FREERTOS_SRC_DIR__)
freertos_plus_src_dir = os.path.join(freertos_lts, __FREERTOS_PLUS_SRC_DIR__)
@@ -167,6 +242,7 @@ def parse_arguments():
parser.add_argument('-c', '--compiler', default='arm-none-eabi-gcc', help='Compiler to use.')
parser.add_argument('-s', '--sizetool', default='arm-none-eabi-size', help='Size tool to use.')
parser.add_argument('-d', '--dontclean', action='store_true', help='Do not clean the generated artifacts.')
parser.add_argument('-g', '--generate-report', action='store_true', help='Generate the JSON report.')
return vars(parser.parse_args())
@@ -175,25 +251,62 @@ def parse_arguments():
def main():
args = parse_arguments()
# Generate Makefile.
generate_makefile(args['lts_path'], args['optimization'], args['lib'])
if args['generate_report']:
# Start with an empty JSON report which is populated as sizes are
# calculated.
generated_json_report = {}
# Run make build.
warnings = make(args['compiler'])
# Read the libraries which are to be included in the report.
with open(__REPORT_LIBS_JSON__) as report_libs_file:
report_libs_json_data = json.load(report_libs_file)
report_libs = report_libs_json_data['libraries']
# Run make size.
calculated_sizes = calculate_size(args['sizetool'])
# Read the template to obtain the library report format.
with open(__JSON_REPORT_TEMPLATE__) as json_report_template_handle:
json_report_template_data = json.load(json_report_template_handle)
lib_report_template = json_report_template_data['lib']
# Remove the generated artifacts.
if not args['dontclean']:
clean()
remove_generated_makefile()
# JSON report has sizes for all the libraries and for both O1 and Os
# Optimizations. Therefore, values for --lib and --optimization are
# ignored.
# Compiled objects files for 'O1' optimization needs to be cleaned
# before 'Os' optimization and therefore, value for --dontclean is
# ignored.
for report_lib in report_libs:
lib_name = report_lib['lib_name']
size_description = report_lib['size_description']
# Print the generated warnings.
pretty_print_warnings(warnings)
o1_sizes = calculate_sizes(args['lts_path'],
'O1',
lib_name,
args['compiler'],
args['sizetool'],
False)
# Print the calculated sizes.
pretty_print_sizes(calculated_sizes)
os_sizes = calculate_sizes(args['lts_path'],
'Os',
lib_name,
args['compiler'],
args['sizetool'],
False)
update_json_report(lib_name,
o1_sizes,
os_sizes,
size_description,
lib_report_template,
generated_json_report)
# Write the generated report to the output file.
with open(__GENERATED_JSON_REPORT__, 'w') as json_report_file_handle:
json_report_file_handle.write(json.dumps(generated_json_report, indent=4, sort_keys=True))
else:
calculate_sizes(args['lts_path'],
args['optimization'],
args['lib'],
args['compiler'],
args['sizetool'],
args['dontclean'])
if __name__ == '__main__':

14
tools/memory_estimator/template/report.json.template Normal file
View File

@@ -0,0 +1,14 @@
{
"lib":{
"files":[
],
"total":{
"total_header":"Total estimate (configurations defined <a href=\"https://github.com/FreeRTOS/FreeRTOS/tree/lts-development/tools/memory_estimator\" target=\"_blank\" rel=\"noopener noreferrer\">here</a>)"
},
"column_header":{
"files_column_header":"File",
"files_o1_header":"With -O1 Optimisation",
"files_os_header":"With -Os Optimisation"
}
}
}

32
tools/memory_estimator/template/report_libs.json Normal file
View File

@@ -0,0 +1,32 @@
{
"libraries": [
{
"lib_name":"https",
"size_description":"Code Size of HTTPS (example generated with GCC for ARM Cortex-M)"
},
{
"lib_name":"jobs",
"size_description":"Code Size of AWS IoT Jobs excluding dependencies (example generated with GCC for ARM Cortex-M)"
},
{
"lib_name":"light-mqtt",
"size_description":"Code Size of Lightweight MQTT API (example generated with GCC for ARM Cortex-M)"
},
{
"lib_name":"mqtt",
"size_description":"Code Size of MQTT (example generated with GCC for ARM Cortex-M)"
},
{
"lib_name":"ota-http",
"size_description":"Code Size of OTA with HTTP excluding dependencies (example generated with GCC for ARM Cortex-M)"
},
{
"lib_name":"ota-mqtt",
"size_description":"Code Size of OTA with MQTT excluding dependencies (example generated with GCC for ARM Cortex-M)"
},
{
"lib_name":"shadow",
"size_description":"Code Size of AWS IoT Device Shadow excluding dependencies (example generated with GCC for ARM Cortex-M)"
}
]
}