ESP32 bulk update sensor data updating rate

Duke Dong on 24 Mar 2022 (Edited on 24 Mar 2022)
Latest activity Reply by Duke Dong on 29 Mar 2022

Hello everyone, I am new to ThingSpeak and ESP32, I would like to send IMU sensor acceleration data from my ESP32 to ThingSpeak so that I can use this data for further analysis. I have tried the example code provided in this link: Bulk-Update and this works perfectly fine for me with the default update interval and post interval. I also attached the code here.

#include <Wire.h>
#include <Arduino.h>
#include <Adafruit_MCP4725.h>
#include <Adafruit_Sensor.h>
#include <Adafruit_BNO055.h>
#include <utility/imumaths.h>
#include <WiFi.h>
#include "ThingSpeak.h"
// Define multiplexer address
#define TCAADDR 0x70
// Define imu and dac objects
Adafruit_BNO055 bno = Adafruit_BNO055(55, 0x28);
char jsonBuffer[500] = "[";
char ssid[] = "my ssid";
char pass[] = "my password";
WiFiClient client;
char server[] = "api.thingspeak.com";
unsigned long lastConnectionTime = 0;
unsigned long lastUpdateTime = 0;
const unsigned long postingInterval = 15L * 1000L;
const unsigned long updateInterval = 3L * 100L;
// To choose the multiplexer specific channel (between 0 and 7)
void tcaselect(uint8_t i){
  if (i > 7) return;
Wire.beginTransmission(TCAADDR);
Wire.write(1 << i);
Wire.endTransmission();
}
    void setup() {
      // put your setup code here, to run once:
  Wire.begin();
  Serial.begin(9600);
  while(!Serial);
  delay(1000);
  Serial.println("Serial communication set up");
// set up bno055
tcaselect(2);
if (!bno.begin()){
  Serial.println("Failed to connect to BNO055");
  while (1);
}
while(WiFi.status() != WL_CONNECTED){
  Serial.print("Attempting to connect to wifi");
  WiFi.begin(ssid, pass);
  delay(5000);
}
Serial.println("Connected to WiFi");
printWiFiStatus();
}
void loop() {
  // put your main code here, to run repeatedly.
if (millis() - lastUpdateTime >= updateInterval){
  updatesJson(jsonBuffer);
}
}
void updatesJson(char* jsonBuffer){
  /* JSON format for updates paramter in the API
   *  This examples uses the relative timestamp as it uses the "delta_t". You can 
also provide the absolute timestamp using the "created_at" parameter
 *  instead of "delta_t".
 *   "[{\"delta_t\":0,\"field1\":-70},{\"delta_t\":3,\"field1\":-66}]"
 */
// Format the jsonBuffer as noted above
strcat(jsonBuffer,"{\"delta_t\":");
unsigned long deltaT = (millis() - lastUpdateTime)/1000;
size_t lengthT = String(deltaT).length();
char temp[4];
String(deltaT).toCharArray(temp,lengthT+1);
strcat(jsonBuffer,temp);
strcat(jsonBuffer,",");
tcaselect(2);
sensors_event_t orientationData , angVelocityData , linearAccelData, magnetometerData, accelerometerData, gravityData;
bno.getEvent(&orientationData, Adafruit_BNO055::VECTOR_EULER);
bno.getEvent(&angVelocityData, Adafruit_BNO055::VECTOR_GYROSCOPE);
bno.getEvent(&linearAccelData, Adafruit_BNO055::VECTOR_LINEARACCEL);
float rssi = linearAccelData.acceleration.x; 
strcat(jsonBuffer, "\"field1\":");
lengthT = String(rssi).length();
String(rssi).toCharArray(temp,lengthT+1);
strcat(jsonBuffer,temp);
strcat(jsonBuffer,"},");
// If posting interval time has reached 15s, update the ThingSpeak channel with your 
data
if (millis() - lastConnectionTime >=  postingInterval) {
      size_t len = strlen(jsonBuffer);
      jsonBuffer[len-1] = ']';
      httpRequest(jsonBuffer);
}
lastUpdateTime = millis(); // Update the last update time
}
// Updates the ThingSpeakchannel with data
void httpRequest(char* jsonBuffer) {
/* JSON format for data buffer in the API
 *  This examples uses the relative timestamp as it uses the "delta_t". You can also 
provide the absolute timestamp using the "created_at" parameter
 *  instead of "delta_t".
 *   "{\"write_api_key\":\"YOUR-CHANNEL-WRITEAPIKEY\",\"updates\": 
[{\"delta_t\":0,\"field1\":-60},{\"delta_t\":15,\"field1\":200}, 
{\"delta_t\":15,\"field1\":-66}]
 */
// Format the data buffer as noted above
char data[500] = "{\"write_api_key\":\"59IISQVOXV5GGH0D\",\"updates\":"; // Replace 
YOUR-CHANNEL-WRITEAPIKEY with your ThingSpeak channel write API key
strcat(data,jsonBuffer);
strcat(data,"}");
// Close any connection before sending a new request
client.stop();
String data_length = String(strlen(data)+1); //Compute the data buffer length
Serial.println(data);
// POST data to ThingSpeak
if (client.connect(server, 80)) {
  client.println("POST /channels/1683708/bulk_update.json HTTP/1.1"); // Replace 
YOUR-CHANNEL-ID with your ThingSpeak channel ID
  client.println("Host: api.thingspeak.com");
  client.println("User-Agent: mw.doc.bulk-update (Arduino ESP8266)");
  client.println("Connection: close");
  client.println("Content-Type: application/json");
  client.println("Content-Length: "+data_length);
  client.println();
  client.println(data);
}
else {
  Serial.println("Failure: Failed to connect to ThingSpeak");
}
delay(250); //Wait to receive the response
client.parseFloat();
String resp = String(client.parseInt());
Serial.println("Response code:"+resp); // Print the response code. 202 indicates that 
the server has accepted the response
jsonBuffer[0] = '['; //Reinitialize the jsonBuffer for next batch of data
jsonBuffer[1] = '\0';
lastConnectionTime = millis(); //Update the last conenction time
}
void printWiFiStatus() {
  // Print the SSID of the network you're attached to:
  Serial.print("SSID: ");
  Serial.println(WiFi.SSID());
// Print your device IP address:
IPAddress ip = WiFi.localIP();
Serial.print("IP Address: ");
Serial.println(ip);
// Print the received signal strength:
long rssi = WiFi.RSSI();
Serial.print("signal strength (RSSI):");
Serial.print(rssi);
Serial.println(" dBm");
}

However, when I tried to change the update interval to 0.3s (30Hz), the following error massage was given in the Arduino monitor: Guru Meditation Error: Core 1 panic'ed (LoadProhibited). Exception was unhandled.

Could someone please help me with this issue. If anything else is needed, I will update immediately.

Many thanks

Christopher Stapels
Christopher Stapels on 24 Mar 2022

ThingSpeak supports time resolution up to 1s. I suspect the request is failing due to the time resolution you selected. You might do some translation on the device and average all the values within a second.

Alternatively, you can use a different field to store timestamps with higher accuracy, but you will have to translate what the actual timestamp means. For example, you can use timestamps from a year ago, and each second equals one millisecond in your data.

Duke Dong
Duke Dong on 24 Mar 2022

Hi Chris,

Thanks for the editing and your reply. Yes the time issue is right, the delt_t in the json code should not be 0 as required. But because the time is set to be an integer so it’s always 0 when frequency is high. I have solved this problem now.

The other issue is: I would like to stream the data in a higher rate, like 30Hz. But then the message will exceed the maximum number allowed. Do you have any suggestions on how to achieve this?

Many thanks

Christopher Stapels
Christopher Stapels on 24 Mar 2022 (Edited on 24 Mar 2022)

There are 8 fields, if you put one measurement in each field, you can get to 8 Hz. There is also a status field (and three locations fields but those have limited range allowed) that you can put a number into.

You can also put multiple measurements in one field. In these cases you will have to do some programming in a MATLAB visualization to reconstitute the data. For example, if your measurement is 8 digits wide, and you use a delimiter, then you can fit 28 measurements in a single field. If you use all eight fields, then you can get to 224 Hz. If you drop the precision, you can get higher frequency.

I'm not sure what license type you have, but paid users can put 15x more data per update. You could also separate the data into multiple channels to get a higher frequency. Once again, you would have to reconstitute the data, but that's easy to do in the built in MATLAB interface.

Last, as I said above, you can do some preprocessing on the edge device. This will result in some loss of information, but if you choose the features that you save carefully, you can get good results.

Duke Dong
Duke Dong on 29 Mar 2022

Hello,

I have tried your way and I now know how to bulk update to multiple fields. But now my question is: how can I upload each data segmentation to each field? Suppose my sensor rate is 30hz, then 15s interval would give me 450 data points. If i want to use 8 fields, then each field should have 60 data points per upload. How can I get each 60 data points? By creating another data buffer maybe?

Thank you.

Christopher Stapels
Christopher Stapels on 29 Mar 2022 (Edited on 29 Mar 2022)

If your sensor rate is 30 Hz, you will need to spread 30 values out over each 1 second interval. You can only bulk post once every 15 seconds (I'd suggest using 30 seconds to be safe).

Here would be the suggested structure m1 is measurement 1, and so on
datetime   f1    f2     f3    f4     f5    f6     f7     f8 
1         m1-4  m5-m8  m9-m12 ...  ...                   m29-m30
2         m31-m34 .. .. .. ... .. ..                     m59-m60
3 ....
and so on.

For each field you can use whatever your favorite delimiter is. Comma, pipe, slash, period etc. I think the bulk update uses comma and pipe so you might want to choose something else

"created_at": "2022-03-05 12:12:12 -0500", "field1": "m1/m2/m3/m4", "field2": "m5/m6/m7/m8", "field3": "m9/m10/m11/m12", "field4": "m13/m14/m15/m16", "field5": "m17/m18/m19/m20", "field6": "m21/m22/m23/m24", "field7": "m25/m26/m27/m28", "field8": "m29/m30",

Duke Dong
Duke Dong on 29 Mar 2022 (Edited on 29 Mar 2022)

I have tried to upload multiple values in a single field as well. I use slash to seperate each value like this:

    // field1 buffer 
    strcat(jsonBuffer, "\"field1\":");
    strcat(jsonBuffer, "\"");
    float rssi = dataBuffer[0]; 
    lengthT = String(rssi).length();
    String(rssi).toCharArray(temp,lengthT+1);
    strcat(jsonBuffer,temp);
    strcat(jsonBuffer, "/"); // I use this slash to seperate each value
    rssi = dataBuffer[1]; 
    lengthT = String(rssi).length();
    String(rssi).toCharArray(temp,lengthT+1);
    strcat(jsonBuffer,temp);
    strcat(jsonBuffer, "\"");
    strcat(jsonBuffer, ",");

Thank you for your support!

Christopher Stapels
Christopher Stapels on 29 Mar 2022

Can you show the output from the serial monitor so I can see how it builds? Are you using the regular write endpoint for that or the bulk endpoint?

Duke Dong
Duke Dong on 29 Mar 2022

I think I am using the bulk endpoint. Here is what I have in the monitor when the length of the json code is < 1000 (when it's functioning):

when the length is greater than 1000, the error message is shown. I have searched the reason, it is because of the buffer overflow. I have tried to add extra space in my buffer but it won't solve the problem.

Christopher Stapels
Christopher Stapels on 29 Mar 2022

From the measurement shown, your values are a fixed length. If that is always true, you don't even need the delimiter and that would save more space. I'd keep up on google to find what to do about the buffer size issue, im not sure if I solved it or just avoided it (I do remember reaching it once or twice.)

Duke Dong
Duke Dong on 29 Mar 2022

Thank you very much! I will attach my code so far here and try to solve the problem as well.

In terms of indexing the sensor value, so the sensor is feeding me the value constantly (~300Hz) and I assigned it to a variable "accelX".

sensors_event_t orientationData , angVelocityData , 
linearAccelData, magnetometerData, accelerometerData, gravityData;
bno.getEvent(&orientationData, Adafruit_BNO055::VECTOR_EULER);
bno.getEvent(&angVelocityData, Adafruit_BNO055::VECTOR_GYROSCOPE);
bno.getEvent(&linearAccelData, Adafruit_BNO055::VECTOR_LINEARACCEL);
float accelX = linearAccelData.acceleration.x;

And then, I will need to update the field1 buffer with the first 4 readings from the sensor. But since there is no slicing operation in arduino, how can I do it? What should I put here in this line of code:

strcat(jsonBuffer, "\"field1\":");
strcat(jsonBuffer, "\"");
float rssi = dataBuffer[0]; // assign the first reading to field1 buffer ???
lengthT = String(rssi).length();
String(rssi).toCharArray(temp,lengthT+1);
strcat(jsonBuffer,temp);

I attached my code here as well.

Thank you very much.

Duke Dong
Duke Dong on 29 Mar 2022

Thank you for your update sir. I am a bit confused here. So the sensor is streaming the data in a much faster rate (~300Hz),but I only need 30Hz for my task. So how should I get each measurement m1, m2, m3, etc? Like how can I index those values?

Christopher Stapels
Christopher Stapels on 29 Mar 2022

You will need to down sample the sensor output somehow yourself. I suggest averaging or just taking every tenth value. I'm not clear why it would be hard to index the values. Don't they come in order chronologically?

Duke Dong
Duke Dong on 24 Mar 2022

Thank you for your help. I will give it a try tomorrow.