Testing for high volume ~12G file looks ok

c-code/binary_storage.c

@@ -4,6 +4,9 @@
 #include <sys/time.h>
 #include "binary_storage.h"
 
+#define INITIAL_SZ 500
+#define MAX_DBL_SZ 500000
+
 
 uint64_t bswap_d(double a){
    uint64_t *b = (uint64_t *)&a;
@@ -13,7 +16,7 @@ uint64_t bswap_d(double a){
 BinaryStorage bsOpen(char* filename){
    BinaryStorage storage;
    storage.file = fopen(filename,"wb");
-   storage.buf_size = 500;
+   storage.buf_size = INITIAL_SZ;
    storage.con_idx = 0;
    storage.contents = (uint64_t*) malloc(sizeof(uint64_t)*storage.buf_size);
 
@@ -28,6 +31,17 @@ void close_record(BinaryStorage *bs){
    /* end of record */
   fputc(BS_ENDRCRD,bs->file);
 
+  if(bs->con_idx >= bs->buf_size){
+    size_t oldsz =  bs->buf_size;
+    size_t newsz = 2*oldsz;
+    uint64_t* temp = (uint64_t*) malloc(sizeof(uint64_t)*newsz);
+    memcpy(temp,bs->contents,sizeof(uint64_t)*oldsz);
+    free(bs->contents);
+    bs->contents = temp;
+    bs->buf_size = newsz;
+    printf("Buffer size was doubled, new size is %d\n",newsz);
+  }
+
   bs->contents[bs->con_idx]=ftell(bs->file);
   bs->con_idx++;
 }
@@ -43,7 +57,7 @@ int write_meta(BinaryStorage *bs, char* name, TSTMP timestamp, TAGS *tags, char*
   uint8_t i=0;
   uint32_t tag_length=0;
   if(tags!=NULL){
-      for(i=0;i<tags->num;i++){
+      for(i=0;i<tags->count;i++){
         tag_length+=strlen(tags->data[i])+2;
       }
   }
@@ -65,7 +79,7 @@ int write_meta(BinaryStorage *bs, char* name, TSTMP timestamp, TAGS *tags, char*
 
 /* tags */
   if(tags!=NULL){
-      for(i=0;i<tags->num;i++){
+      for(i=0;i<tags->count;i++){
         unsigned char tag_len = strlen(tags->data[i]);
         fputc(BS_TAG,bs->file);
         fputc(tag_len,bs->file);

c-code/binary_storage.h

@@ -46,8 +46,9 @@
 
 
 typedef struct tagset {
+  size_t count;
   char **data;
-  size_t num;
+
 } TAGS;
 
 typedef struct timestamp {

c-code/test_code.c

@@ -3,32 +3,42 @@
 #include <binary_storage.h>
 
 int main(){
-   struct timeval tp;
-   long int ms;
-   BinaryStorage storage = bsOpen("test_c.bin");
 
-   char *tag1 = "TAG1";
-   char *data[]={"MEGA_TAG",tag1};
-   TAGS tags={.data=data, .num=2}; 
+   BinaryStorage storage = bsOpen("test_c.bin");
+   TAGS tags = {.data=NULL, .count = 0}; //empty
 
    int i=0;
-   for(i=0;i<10;i++){
+   for(i=0;i<30000;i++){
+      struct timeval tp;
+      long int ms;
       gettimeofday(&tp, NULL);
       ms = tp.tv_sec * 1000 + tp.tv_usec / 1000;
       TSTMP timestamp = {.seconds = tp.tv_sec, .nanos=tp.tv_usec*1000};
 
+      double wf[100000];
+      wf[0] = 1.0;
+      wf[1] = 2.0;
+
       DBL value;
       double x;
       char *description;
+      char *tt[10];
 
       if(i%2==0){
           x = 3.1415*i;
           value = (DBL){.count = 1, .data=&x};
           description = "double scalar";
-      } else {
-          double wf[] = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0};
+          tt[0]="SCALAR";
+          tt[1]="DOUBLE";
+          tags = (TAGS){.data=tt, .count = 2};
+      }
+      else {
+
           value = (DBL){.count = sizeof(wf)/8, .data=wf};
           description = "double[]";
+          tt[0]="ARRAY";
+          tt[1]="DOUBLE";
+          tags = (TAGS){.data=tt, .count = 2};
       }
 
       bsWriteDBL(&storage,"Test_Record", timestamp, value, &tags, description);