Squashed 'connectivity/libraries/nanostack-libservice/' changes from 67d7b93ec6..a526cc3f41

a526cc3f41 Merge pull request ARMmbed#97 from PelionIoT/sync_from_mbed_os
2f82569918 Add license notice to Arm copyrighted source files
ad7631083a Merge pull request ARMmbed#96 from PelionIoT/heap_lite_stats
18b742b2c7 Heap stat's enabled allways by default.
9b4cbaf101 Added extended block list to dynamic memory tracker (ARMmbed#95)
8ff6d31abb Added dynamic memory tracker and hooks to dynmem library (ARMmbed#93)
cf2b133082 Merge pull request ARMmbed#92 from LDong-Arm/add_present_flag
9eb6f2dde7 Add config to generate macro MBED_CONF_NANOSTACK_LIBSERVICE_PRESENT

git-subtree-dir: connectivity/libraries/nanostack-libservice
git-subtree-split: a526cc3f41bfd0ec6a78e6998b8f8dcc1bd2afc9

Author: Arto Kinnunen

Makefile

@@ -5,6 +5,7 @@ source/libip6string/ip6tos.c \
 source/libip6string/stoip6.c \
 source/libList/ns_list.c \
 source/nsdynmemLIB/nsdynmemLIB.c \
+source/nsdynmemtracker/nsdynmem_tracker_lib.c \
 source/nvmHelper/ns_nvm_helper.c \
 
 LIB := libservice.a

mbed-client-libservice/nsdynmemLIB.h

@@ -37,6 +37,9 @@ extern "C" {
 typedef size_t ns_mem_block_size_t; //external interface unsigned heap block size type
 typedef size_t ns_mem_heap_size_t; //total heap size type.
 
+// Can be used to enable tracking of dynamic memory allocations
+#include "nsdynmem_tracker.h"
+
 /*!
  * \enum heap_fail_t
  * \brief Dynamically heap system failure call back event types.
@@ -64,7 +67,6 @@ typedef struct mem_stat_t {
     uint32_t heap_alloc_fail_cnt;               /**< Counter for Heap allocation fail. */
 } mem_stat_t;
 
-
 typedef struct ns_mem_book ns_mem_book_t;
 
 /**
@@ -99,7 +101,9 @@ extern int ns_dyn_mem_region_add(void *region_ptr, ns_mem_heap_size_t region_siz
   * \return 0, Free OK
   * \return <0, Free Fail
   */
+#if NSDYNMEM_TRACKER_ENABLED!=1
 extern void ns_dyn_mem_free(void *heap_ptr);
+#endif
 
 /**
   * \brief Allocate temporary data.
@@ -111,7 +115,9 @@ extern void ns_dyn_mem_free(void *heap_ptr);
   * \return 0, Allocate Fail
   * \return >0, Pointer to allocated data sector.
   */
+#if NSDYNMEM_TRACKER_ENABLED!=1
 extern void *ns_dyn_mem_temporary_alloc(ns_mem_block_size_t alloc_size);
+#endif
 
 /**
   * \brief Allocate long period data.
@@ -123,7 +129,9 @@ extern void *ns_dyn_mem_temporary_alloc(ns_mem_block_size_t alloc_size);
   * \return 0, Allocate Fail
   * \return >0, Pointer to allocated data sector.
   */
+#if NSDYNMEM_TRACKER_ENABLED!=1
 extern void *ns_dyn_mem_alloc(ns_mem_block_size_t alloc_size);
+#endif
 
 /**
   * \brief Get pointer to the current mem_stat_t set via ns_dyn_mem_init.

mbed-client-libservice/nsdynmem_tracker.h

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2020 ARM Limited. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * \file nsdynmem_tracker.h
+ * \brief Dynamical Memory Tracker definitions to override the default NS dynamic memory functionality
+ * Provides tracking and tracing of dynamic memory blocks
+ */
+
+#ifndef NSDYNMEM_TRACKER_H_
+#define NSDYNMEM_TRACKER_H_
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#if NSDYNMEM_TRACKER_ENABLED==1
+
+#define ns_dyn_mem_free(block) ns_dyn_mem_tracker_dyn_mem_free(block, __func__, __LINE__)
+#define ns_dyn_mem_temporary_alloc(alloc_size) ns_dyn_mem_tracker_dyn_mem_temporary_alloc(alloc_size, __func__, __LINE__)
+#define ns_dyn_mem_alloc(alloc_size) ns_dyn_mem_tracker_dyn_mem_alloc(alloc_size, __func__, __LINE__)
+
+void *ns_dyn_mem_tracker_dyn_mem_alloc(ns_mem_heap_size_t alloc_size, const char *function, uint32_t line);
+void *ns_dyn_mem_tracker_dyn_mem_temporary_alloc(ns_mem_heap_size_t alloc_size, const char *function, uint32_t line);
+void ns_dyn_mem_tracker_dyn_mem_free(void *block, const char *function, uint32_t line);
+
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* NSDYNMEM_TRACKER_H_ */
+
+

mbed-client-libservice/nsdynmem_tracker_lib.h

@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2020 ARM Limited. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+/**
+ * \file nsdynmem_tracker_lib.h
+ * \brief Dynamical Memory Tracker library API
+ * Provides tracking and tracing of dynamic memory blocks
+ */
+
+#ifndef NSDYNMEM_TRACKER_LIB_H_
+#define NSDYNMEM_TRACKER_LIB_H_
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#if NSDYNMEM_TRACKER_ENABLED==1
+
+// Memory block structure with caller information
+typedef struct ns_dyn_mem_tracker_lib_mem_blocks_s {
+    void *block;                   /**< Allocated memory block */
+    void *caller_addr;             /**< Caller address */
+    uint32_t size;                 /**< Allocation size */
+    uint32_t total_size;           /**< Total allocation size for all allocations */
+    uint32_t lifetime;             /**< Memory block lifetime in steps (e.g. seconds) */
+    uint32_t ref_count;            /**< Reference count */
+    const char *function;          /**< Caller function */
+    uint16_t line;                 /**< Caller line in module */
+    bool permanent : 1;            /**< Permanent memory block */
+    bool permanent_printed : 1;    /**< Permanent memory block printed */
+} ns_dyn_mem_tracker_lib_mem_blocks_t;
+
+// Extended memory block structure that is used if same caller allocates multiple memory blocks
+typedef struct ns_dyn_mem_tracker_lib_mem_blocks_ext_s {
+    void *block;                   /**< Allocated memory block */
+    void *caller_addr;             /**< Caller address */
+    uint32_t size;                 /**< Allocation size */
+} ns_dyn_mem_tracker_lib_mem_blocks_ext_t;
+
+// Allocator information structure
+typedef struct ns_dyn_mem_tracker_lib_allocators_s {
+    void *caller_addr;             /**< Caller address */
+    uint32_t alloc_count;          /**< Number of allocations */
+    uint32_t total_memory;         /**< Total memory used by allocations */
+    uint32_t min_lifetime;         /**< Shortest lifetime among the allocations */
+    const char *function;          /**< Function name string */
+    uint16_t line;                 /**< Module line */
+} ns_dyn_mem_tracker_lib_allocators_t;
+
+// Memory block array allocator / array size increase allocator
+typedef ns_dyn_mem_tracker_lib_mem_blocks_t *ns_dyn_mem_tracker_lib_alloc_mem_blocks(ns_dyn_mem_tracker_lib_mem_blocks_t *blocks, uint16_t *mem_blocks_count);
+// Extended memory block array allocator / array size increase allocator
+typedef ns_dyn_mem_tracker_lib_mem_blocks_ext_t *ns_dyn_mem_tracker_lib_alloc_mem_blocks_ext(ns_dyn_mem_tracker_lib_mem_blocks_ext_t *blocks, uint32_t *mem_blocks_count);
+// Extended memory block array index hash function to get memory block (allocation/search start) index from block address
+typedef uint32_t ns_dyn_mem_tracker_lib_mem_block_index_hash(void *block, uint32_t ext_mem_blocks_count);
+
+typedef struct ns_dyn_mem_tracker_lib_conf_s {
+    ns_dyn_mem_tracker_lib_mem_blocks_t *mem_blocks;                    /**< Memory blocks array, if NULL calls allocator on init */
+    ns_dyn_mem_tracker_lib_mem_blocks_ext_t *ext_mem_blocks;            /**< Extended memory blocks array, if NULL calls allocator on init */
+    ns_dyn_mem_tracker_lib_allocators_t *top_allocators;                /**< Top allocators array */
+    ns_dyn_mem_tracker_lib_allocators_t *permanent_allocators;          /**< Permanent allocators */
+    ns_dyn_mem_tracker_lib_allocators_t *to_permanent_allocators;       /**< To permanent allocators */
+    ns_dyn_mem_tracker_lib_allocators_t *max_snap_shot_allocators;      /**< Snap shot of maximum memory used by allocators */
+    ns_dyn_mem_tracker_lib_alloc_mem_blocks *alloc_mem_blocks;          /**< Memory block array allocator / array size increase allocator */
+    ns_dyn_mem_tracker_lib_alloc_mem_blocks_ext *ext_alloc_mem_blocks;  /**< Extended memory block array allocator / array size increase allocator */
+    ns_dyn_mem_tracker_lib_mem_block_index_hash *block_index_hash;      /**< Hash function to get memory block index from block address */
+    uint32_t allocated_memory;                                          /**< Currently allocated memory */
+    uint16_t mem_blocks_count;                                          /**< Number of entries in memory blocks array */
+    uint32_t ext_mem_blocks_count;                                      /**< Number of entries in extended memory blocks array */
+    uint16_t last_mem_block_index;                                      /**< Last memory block in memory blocks array */
+    uint16_t top_allocators_count;                                      /**< Top allocators array count */
+    uint16_t permanent_allocators_count;                                /**< Permanent allocators array count */
+    uint16_t to_permanent_allocators_count;                             /**< To permanent allocators array count */
+    uint16_t max_snap_shot_allocators_count;                            /**< Snap shot of maximum memory used by allocators array count */
+    uint16_t to_permanent_steps_count;                                  /**< How many steps before moving block to permanent allocators list */
+} ns_dyn_mem_tracker_lib_conf_t;
+
+int8_t ns_dyn_mem_tracker_lib_alloc(ns_dyn_mem_tracker_lib_conf_t *conf, void *caller_addr, const char *function, uint32_t line, void *block, uint32_t alloc_size);
+int8_t ns_dyn_mem_tracker_lib_free(ns_dyn_mem_tracker_lib_conf_t *conf, void *caller_addr, const char *function, uint32_t line, void *block);
+void ns_dyn_mem_tracker_lib_step(ns_dyn_mem_tracker_lib_conf_t *conf);
+int8_t ns_dyn_mem_tracker_lib_allocator_lists_update(ns_dyn_mem_tracker_lib_conf_t *conf);
+void ns_dyn_mem_tracker_lib_max_snap_shot_update(ns_dyn_mem_tracker_lib_conf_t *conf);
+
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* NSDYNMEM_TRACKER_LIB_H_ */
+
+

mbed_lib.json

@@ -1,3 +1,11 @@
 {
-    "name": "nanostack-libservice"
+    "name": "nanostack-libservice",
+    "macros": ["NSDYNMEM_TRACKER_ENABLED=MBED_CONF_NANOSTACK_LIBSERVICE_NSDYNMEM_TRACKER_ENABLED"],
+    "config": {
+        "present": 1,
+        "nsdynmem-tracker-enabled": {
+            "help": "Use to enable dynamic memory tracker",
+            "value": 0
+        }
+    }
 }

source/nsdynmemLIB/nsdynmemLIB.c

@@ -15,6 +15,7 @@
  */
 #include <stdint.h>
 #include <string.h>
+#undef NSDYNMEM_TRACKER_ENABLED
 #include "nsdynmemLIB.h"
 #include "platform/arm_hal_interrupt.h"
 #include <stdlib.h>
@@ -47,6 +48,7 @@ struct ns_mem_book {
     ns_mem_heap_size_t temporary_alloc_heap_limit;   /* Amount of reserved heap temporary alloc can't exceed */
 };
 
+static mem_stat_t default_stats;
 static ns_mem_book_t *default_book; // heap pointer for original "ns_" API use
 
 // size of a hole_t in our word units
@@ -162,12 +164,14 @@ ns_mem_book_t *ns_mem_init(void *heap, ns_mem_heap_size_t h_size,
     ns_list_init(&book->holes_list);
     ns_list_add_to_start(&book->holes_list, hole_from_block_start(book->heap_main[0]));
 
-    book->mem_stat_info_ptr = info_ptr;
-    //RESET Memory by Hea Len
     if (info_ptr) {
-        memset(book->mem_stat_info_ptr, 0, sizeof(mem_stat_t));
-        book->mem_stat_info_ptr->heap_sector_size = book->heap_size;
+        book->mem_stat_info_ptr = info_ptr;
+    } else {
+        book->mem_stat_info_ptr = &default_stats;
     }
+    //RESET Memory by Heap Len
+    memset(book->mem_stat_info_ptr, 0, sizeof(mem_stat_t));
+    book->mem_stat_info_ptr->heap_sector_size = book->heap_size;
     book->temporary_alloc_heap_limit = book->heap_size / 100 * (100 - TEMPORARY_ALLOC_FREE_HEAP_THRESHOLD);
 #endif
     //There really is no support to standard malloc in this library anymore
@@ -234,9 +238,7 @@ int ns_mem_region_add(ns_mem_book_t *book, void *region_ptr, ns_mem_heap_size_t
     // adjust total heap size with new hole
     book->heap_size += region_size;
 
-    if (book->mem_stat_info_ptr) {
-        book->mem_stat_info_ptr->heap_sector_size = book->heap_size;
-    }
+    book->mem_stat_info_ptr->heap_sector_size = book->heap_size;
 
     // adjust temporary allocation limits to match new heap
     book->temporary_alloc_heap_limit = book->heap_size / 100 * (100 - TEMPORARY_ALLOC_FREE_HEAP_THRESHOLD);
@@ -265,8 +267,8 @@ int ns_mem_set_temporary_alloc_free_heap_threshold(ns_mem_book_t *book, uint8_t
 #ifndef STANDARD_MALLOC
     ns_mem_heap_size_t heap_limit = 0;
 
-    if (!book || !book->mem_stat_info_ptr) {
-        // no book or mem_stats
+    if (!book) {
+        // no book
         return -1;
     }
 
@@ -304,25 +306,25 @@ extern int ns_dyn_mem_set_temporary_alloc_free_heap_threshold(uint8_t free_heap_
 #ifndef STANDARD_MALLOC
 static void dev_stat_update(mem_stat_t *mem_stat_info_ptr, mem_stat_update_t type, ns_mem_block_size_t size)
 {
-    if (mem_stat_info_ptr) {
-        switch (type) {
-            case DEV_HEAP_ALLOC_OK:
-                mem_stat_info_ptr->heap_sector_alloc_cnt++;
-                mem_stat_info_ptr->heap_sector_allocated_bytes += size;
-                if (mem_stat_info_ptr->heap_sector_allocated_bytes_max < mem_stat_info_ptr->heap_sector_allocated_bytes) {
-                    mem_stat_info_ptr->heap_sector_allocated_bytes_max = mem_stat_info_ptr->heap_sector_allocated_bytes;
-                }
-                mem_stat_info_ptr->heap_alloc_total_bytes += size;
-                break;
-            case DEV_HEAP_ALLOC_FAIL:
-                mem_stat_info_ptr->heap_alloc_fail_cnt++;
-                break;
-            case DEV_HEAP_FREE:
-                mem_stat_info_ptr->heap_sector_alloc_cnt--;
-                mem_stat_info_ptr->heap_sector_allocated_bytes -= size;
-                break;
-        }
+
+    switch (type) {
+        case DEV_HEAP_ALLOC_OK:
+            mem_stat_info_ptr->heap_sector_alloc_cnt++;
+            mem_stat_info_ptr->heap_sector_allocated_bytes += size;
+            if (mem_stat_info_ptr->heap_sector_allocated_bytes_max < mem_stat_info_ptr->heap_sector_allocated_bytes) {
+                mem_stat_info_ptr->heap_sector_allocated_bytes_max = mem_stat_info_ptr->heap_sector_allocated_bytes;
+            }
+            mem_stat_info_ptr->heap_alloc_total_bytes += size;
+            break;
+        case DEV_HEAP_ALLOC_FAIL:
+            mem_stat_info_ptr->heap_alloc_fail_cnt++;
+            break;
+        case DEV_HEAP_FREE:
+            mem_stat_info_ptr->heap_sector_alloc_cnt--;
+            mem_stat_info_ptr->heap_sector_allocated_bytes -= size;
+            break;
     }
+
 }
 
 static ns_mem_word_size_t convert_allocation_size(ns_mem_book_t *book, ns_mem_block_size_t requested_bytes)
@@ -363,7 +365,7 @@ static void *ns_mem_internal_alloc(ns_mem_book_t *book, const ns_mem_block_size_
         return NULL;
     }
 
-    if (book->mem_stat_info_ptr && direction == 1) {
+    if (direction == 1) {
         if (book->mem_stat_info_ptr->heap_sector_allocated_bytes > book->temporary_alloc_heap_limit) {
             /* Not enough heap for temporary memory allocation */
             dev_stat_update(book->mem_stat_info_ptr, DEV_HEAP_ALLOC_FAIL, 0);
@@ -441,14 +443,14 @@ static void *ns_mem_internal_alloc(ns_mem_book_t *book, const ns_mem_block_size_
     block_ptr[1 + data_size] = data_size;
 
 done:
-    if (book->mem_stat_info_ptr) {
-        if (block_ptr) {
-            //Update Allocate OK
-            dev_stat_update(book->mem_stat_info_ptr, DEV_HEAP_ALLOC_OK, (data_size + 2) * sizeof(ns_mem_word_size_t));
-        } else {
-            //Update Allocate Fail, second parameter is used for stats
-            dev_stat_update(book->mem_stat_info_ptr, DEV_HEAP_ALLOC_FAIL, 0);
-        }
+
+
+    if (block_ptr) {
+        //Update Allocate OK
+        dev_stat_update(book->mem_stat_info_ptr, DEV_HEAP_ALLOC_OK, (data_size + 2) * sizeof(ns_mem_word_size_t));
+    } else {
+        //Update Allocate Fail, second parameter is used for stats
+        dev_stat_update(book->mem_stat_info_ptr, DEV_HEAP_ALLOC_FAIL, 0);
     }
     platform_exit_critical();
 
@@ -619,10 +621,8 @@ void ns_mem_free(ns_mem_book_t *book, void *block)
             heap_failure(book, NS_DYN_MEM_HEAP_SECTOR_CORRUPTED);
         } else {
             ns_mem_free_and_merge_with_adjacent_blocks(book, ptr, size);
-            if (book->mem_stat_info_ptr) {
-                //Update Free Counter
-                dev_stat_update(book->mem_stat_info_ptr, DEV_HEAP_FREE, (size + 2) * sizeof(ns_mem_word_size_t));
-            }
+            //Update Free Counter
+            dev_stat_update(book->mem_stat_info_ptr, DEV_HEAP_FREE, (size + 2) * sizeof(ns_mem_word_size_t));
         }
     }