/****************************************************************************/
/*                                                                          */
/*                              Dump_to_JSON.c                              */
/*                                                                          */
/****************************************************************************/
/****************************************************************************/
/* This is a filter program that reads a Dump format data file and writes   */
/* out the same data in json format.                                        */
/*                                                                          */
/* Usage:                                                                   */
/*  Dump_to_JSON [-s] [-e | -h] [-o] [-c] [-a]                              */
/*                                          [<] DumpFile > JSONFile         */
/*      [-s YYYYMMDDHH]     Time of the first record included. If missing   */
/*                          then start of file is assumed.                  */
/*      [-e YYYYMMDDHH]     Time of the record not included anymore. If     */
/*                          missing then end of file is assumed.            */
/*      [-h HH]             Number of hours included. Either -e or -h can   */
/*                          be specified, not both.                         */
/*      [-o 'Station list'] List of stations delimited by aposthropes       */
/*                          If missing then all stations included.          */
/*                          Stations are identified by three-letter code    */
/*                          and separated by exactly one space.             */
/*      [-c 'CompList']     Components to be written. Possible values are   */
/*                          XYZHDF. Default value is XYZ.                   */
/*      [-a AverPeriod]     Don't write all data values but averages over   */
/*                          AverPeriod.                                     */
/*      [-b BaseLineInfo]   Determines the baseline values stored in the    */
/*                          JSON file.                                      */
/*                          BaseLineInfo may have the following two values: */
/*                          1. H1-H2  The baseline is determined as the     */
/*                             average field value from hour H1 to hour H2  */
/*                             (H2 is not included). The hour is counted    */
/*                             from the start time of the data file.        */
/*                             E.g  -s 94120106 -b 6-7  implies that the    */
/*                             baseline is the one hour average from        */
/*                             94120112 to 94120113.                        */
/*                          2. QN  The baseline is determined as the        */
/*                             average of the quietest N hour interval in   */
/*                             the datafile including all stations.         */
/*                             E.g -b Q2 implies that the program computes  */
/*                             the sum of max-min values of all stations    */
/*                             for the intervals 0-2,1-3,2-4, ... and uses  */
/*                             the interval with smallest variations to     */
/*                             compute the baseline value.                  */
/*                          If -b is missing then the baseline is set to    */
/*                          the average value of the field over the whole   */
/*                          period of the plot.                             */
/*      DumpFile            Name of Dump-format file.                       */
/*      JSONFile            Name of JSON file.                              */
/*                                                                          */
/****************************************************************************/
/****************************************************************************/
/*                            Lasse Hakkinen                                */
/*                    Finnish Meteorological Institute                      */
/*                      Geophysical Research Division                       */
/*                              P.O.Box 503                                 */
/*                      FIN-00101, Helsinki, Finland                        */
/*                      e-mail: Lasse.Hakkinen@fmi.fi                       */
/*                      phone : (+358)-29-5394634                           */
/*                                                                          */
/*                      version 1.01        03.01.2020                      */
/****************************************************************************/
/****************************************************************************/
/*  Version history:                                                        */
/*                                                                          */
/*  1.01 03.01.2020 Date strings must be YYYYMMDD (four digits for year)    */
/*                  Replaced StrToSecs -> StrToSecsC and                    */
/*                  SecsToStr -> SecsToStrC.                                */
/*  1.0  09.02.2016 First official release                                  */
/****************************************************************************/

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <limits.h>
#include "Usage.h"
#include "MagnData.h"
#include "Dump.h"

char *version = "1.01";
char *date = "03.01.2020";

#define HeaderLineCount 4
#define UsageLineCount 38

char *HeaderText[HeaderLineCount] = {
"**************************************************************************",
"This program reads an Dump-format magnetometer data file and writes out   ",
"the data for given components and stations in JSON format (TEXT).         ",
"**************************************************************************",
};

char *UsageText[UsageLineCount] = {
" [-s] [-e | -h] [-o] [-c] [-a] [-b] DumpFile > JSONFile",
"       [-s YYYYMMDDHH]     Time of the first record included. If missing   ",
"                           then start of file is assumed.                  ",
"       [-e YYYYMMDDHH]     Time of the record not included anymore. If     ",
"                           missing then end of file is assumed.            ",
"       [-h HH]             Number of hours included. Either -e or -h can   ",
"                           be specified, not both.                         ",
"       [-o 'Station list'] List of stations delimited by aposthropes       ",
"                           If missing then all stations included.          ",
"                           Stations are identified by three-letter code    ",
"                           and separated by exactly one space.             ",
"       [-c 'CompList']     List of components to be printed. Possible      ",
"                           values for the components are X,Y,Z,H,D,F.      ",
"                           Default value for CompList is XYZ.              ",
"       [-a AverPeriod]     Write data values as averages over AverPeriod.  ",
"       [-b BaseLineInfo]   Determines the baseline values stored in the    ",
"                           JSON file.                                      ",
"                           BaseLineInfo may have the following two values: ",
"                           1. H1-H2  The baseline is determined as the     ",
"                              average field value from hour H1 to hour H2  ",
"                              (H2 is not included). The hour is counted    ",
"                              from the start time of the data file.        ",
"                              E.g  -s 94120106 -b 6-7  implies that the    ",
"                              baseline is the one hour average from        ",
"                              94120112 to 94120113.                        ",
"                           2. QN  The baseline is determined as the        ",
"                              average of the quietest N hour interval in   ",
"                              the datafile including all stations.         ",
"                              E.g -b Q2 implies that the program computes  ",
"                              the sum of max-min values of all stations    ",
"                              for the intervals 0-2,1-3,2-4, ... and uses  ",
"                              the interval with smallest variations to     ",
"                              compute the baseline value.                  ",
"                           If -b is missing then the baseline is set to    ",
"                           the average value of the field over the whole   ",
"                           period of the plot.                             ",
"       DumpFile            Name of Dump-format file.                       ",
"       JSONFile            Name of JSON format file.                       ",
};


/*--------------------------------------------------------------------------*/
/*  Get the start and end hours from the BaseStr. The format of BaseStr is  */
/*  HH-HH.                                                                  */
/*--------------------------------------------------------------------------*/

void GetHours(char *BaseStr,long *t0, long *t1)
{
    long i = 0;
    long t = 0;
    long negative = 0;

    if (BaseStr[0] == '-') {
        negative = 1;
        i++;
    }
    while ((BaseStr[i] != '-') && (i < strlen(BaseStr))) {
        t = 10*t + (BaseStr[i]-48);
        i++;
    }
    if (negative) t = -t;
    *t0 = t;

    t = 0;
    negative = 0;
    if (BaseStr[++i] == '-') {
        negative = 1;
        i++;
    }
    while (i < strlen(BaseStr))
        t = 10*t + (BaseStr[i++]-48);

    if (negative) t = -t;
    *t1 = t;
}


/*--------------------------------------------------------------------------*/
/*  Get the hour count from the BaseStr. The format of the string is QHH.   */
/*--------------------------------------------------------------------------*/

long GetQHour(char *BaseStr)
{
    long i = 0;
    long t = 0;

    while (++i < strlen(BaseStr)) t = 10*t + (BaseStr[i]-48);
    return t;
}

/*--------------------------------------------------------------------------*/
/*  Find the baseline value for given station and given field component.    */
/*  If the baseline value is not defined in the parameter file then compute */
/*  the value as the average value over the whole dataplot period.          */
/*--------------------------------------------------------------------------*/

void FindBaselineHours(NetworkPtr Network, char *BaseStr, long *StartHour, long *EndHour)
{
    long t,t0,t1;
    long Hours,Max,Min;
    long MaxMin;
    long Good;
    long sum, count;
    StationPtr station;
    char Component = 'X';   // Use only X component

    // If BaseStr is not defined then the average is computed over whole data period
    if (strlen(BaseStr) == 0) {
        station = Network->StationList;
        *StartHour = 0;
        *EndHour = (station->EndTime - station->StartTime)/3600;
        if (*EndHour == 0) *EndHour = 1;
        return;
    }

    /* Check the BaseStr (-b option) and compute the baseline accordingly. */
    if (BaseStr[0] == 'Q') {            /* Quiet period average */
        Hours = GetQHour(BaseStr);      /* Number of hours      */
        t0 = Network->StationList->StartTime;
        MaxMin = LONG_MAX;
        for (t = Network->StationList->StartTime; t< Network->StationList->EndTime-3600*Hours; t += 3600) {
            sum = 0;
            count = 0;
            station = Network->StationList;
            while (station != NULL) {
                FindMaxMin(station,Component,t,3600*Hours,&Max,&Min);
                Good = FindPercentage(station,Component,t,3600*Hours);
                if ((Max != Min) && (Good > 80)) {
                    sum += Max-Min;
                    count++;
                }
                station = station->Next;
            }
            if (sum/count < MaxMin) {
                MaxMin = sum/count;
                t0 = t;
            }
        }
        *StartHour = (t0-Network->StationList->StartTime)/3600;
        *EndHour   = *StartHour + Hours;
    }
    else {                  /* Average over specified hours H1-H2*/
        GetHours(BaseStr,&t0,&t1);
        *StartHour = t0;
        *EndHour = t1;
    }
}


/*--------------------------------------------------------------------------*/
/*                          The main procedure                              */
/*--------------------------------------------------------------------------*/

int main(int argc, char *argv[])
{
    long params;
    long status    = 0;
    long FileCount = 0;
    long HourCount = 0;
    long AvePeriod = 0;
    long i,Ave,base;
    char FileName[100]    = "";
    char StartTimeStr[20] = "";
    char EndTimeStr[20]   = "";
    char StationStr[200]  = "";
    char ComponentStr[20] = "";
    StationPtr s;
    Time_sec t, StartTime, EndTime, CurrTime, UnixTimeStamp;
    Time_sec DataStartTime, DataEndTime;
    char TimeStr[20];
    char c, cChar;
    char BaselineStr[20] = "";
    long found;
    long StartHour, EndHour;
    long FullDay = 0;
    char InfoLineStr[100] = "";


    Network_struct NETWORK;


    /*==========================*/
    /* Analyse the command line */
    /*==========================*/
    if (argc == 1) {        /* No arguments, show the usage text */
        PrintUsage(argv[0],0,HeaderLineCount,UsageLineCount);
        return OK;
    }

    for (params = 1; params < argc; params++) {
        if (*argv[params] != '-') {
            strcpy(FileName,argv[params]);
            FileCount++;
        } else {
            switch (*(argv[params]+1)) {
                case 's' : strcpy(StartTimeStr,argv[++params]); break;
                case 'e' : strcpy(EndTimeStr,argv[++params]);   break;
                case 'h' : HourCount = atol(argv[++params]);    break;
                case 'o' : strcpy(StationStr,argv[++params]);   break;
                case 'c' : strcpy(ComponentStr,argv[++params]); break;
                case 'a' : AvePeriod = atol(argv[++params]);    break;
                case 'f' : FullDay = 1;                         break;
                case 'b' : strcpy(BaselineStr,argv[++params]);  break;
                case 'p' : /* Do nothing */                     break;
                default  :
                    fprintf(stderr,"\n### %s: \"%s\" is not an option.\n\n",
                            argv[0], argv[params]);
                    PrintUsage(argv[0],1,HeaderLineCount,UsageLineCount);
                    return FAIL;
                    break;
            }
        }
    }

    /*====================================*/
    /* Check the values of the parameters */
    /*====================================*/

    if (FileCount > 1) {
        PrintUsage(argv[0],1,HeaderLineCount,UsageLineCount);
        return FAIL;
    }
    if ((HourCount != 0) && (strlen(EndTimeStr) > 0)) {
        PrintUsage(argv[0],1,HeaderLineCount,UsageLineCount);
        return FAIL;
    }
    if ((HourCount != 0) && (strlen(StartTimeStr) == 0)) {
        PrintUsage(argv[0],1,HeaderLineCount,UsageLineCount);
        return FAIL;
    }
    if (HourCount != 0) {
        SecsToStrC(StrToSecsC(StartTimeStr,0)+3600*HourCount,EndTimeStr);
    }


    /*====================================================*/
    /* Read the data from an Dump format file into memory */
    /*====================================================*/

    status = ReadDump(FileName,&NETWORK,StartTimeStr,EndTimeStr,StationStr);

    if (status != 0) {
        if (status == FileError)
            fprintf(stderr,"Error in reading file %s\n",FileName);
        if (status == OutOfMemory)
            fprintf(stderr,"Out of memory while reading file %s\n",FileName);
        if (status == FileFormatError)
            fprintf(stderr,"%s is not an Dump format file\n",FileName);
        return FAIL;
    }


    /*===============================*/
    /* Check the magnetic components */
    /*===============================*/

    if (strlen(ComponentStr) == 0) {
        strcpy(ComponentStr, NETWORK.StationList->Components);
    }
    else {
        for (i=0;i<strlen(ComponentStr);i++) {
            c = ComponentStr[i];
            if (strchr(NETWORK.StationList->Components, c) == NULL) {
                if (c != 'H' && c != 'D' && c != 'F') {
                    fprintf(stderr,"Component %c not found in data file %s\n", c,FileName);
                    return FAIL;
                }
            }
        }
    }

    FindBaselineHours(&NETWORK, BaselineStr, &StartHour, &EndHour);

    // Extend data to full day boundaries
    if (FullDay == 1) {
        if ((((NETWORK.StationList->EndTime) % 86400) != 0) || (((NETWORK.StationList->StartTime) % 86400) != 0)) {
            StartTime = 86400*((NETWORK.StationList->StartTime)/86400);

            if (((NETWORK.StationList->EndTime) % 86400) != 0)
                EndTime = 86400*((NETWORK.StationList->EndTime)/86400 + 1);
            else
                EndTime = NETWORK.StationList->EndTime;

            ExtendDataArea(NETWORK.StationList, StartTime, EndTime);
        }
    }

    /*==============================================*/
    /* Write the data in json format into stdout    */
    /*==============================================*/

    printf("{");    // Opening json brace
    printf("\n");

    StartTime = (strlen(StartTimeStr) == 0) ? NETWORK.StationList->StartTime : StrToSecsC(StartTimeStr,0);
    SecsToStrC(StartTime,TimeStr);
    printf("\"starttime\":\"%s\",", TimeStr);
    printf("\n");

    EndTime = (strlen(EndTimeStr) == 0) ? NETWORK.StationList->EndTime : StrToSecsC(EndTimeStr,0);
    SecsToStrC(EndTime,TimeStr);
    printf("\"endtime\":\"%s\",", TimeStr);
    printf("\n");

    // Write StartTime and Endtime in unix time stamp (= number of seconds since Jan 1, 1970, 0UT)
    UnixTimeStamp = StrToSecsC("19700101", 0);
    printf("\"starttime_secs\": %ld,", StartTime - UnixTimeStamp);
    printf("\n");
    printf("\"endtime_secs\": %ld,", EndTime - UnixTimeStamp);
    printf("\n");

    if (AvePeriod == 0) AvePeriod = NETWORK.StationList->TimeStep;
    printf("\"timestep\":%d,", AvePeriod);
    printf("\n");

    printf("\"count\":%d,", (EndTime - StartTime)/AvePeriod);
    printf("\n");

    printf("\"stations\":[");
    s = NETWORK.StationList;            // Go through the stations. This is actually unnecessary since there is only one station in a dump file.
    while (s != NULL) {
        printf("\"%s\"", s->StationID);
        s = s->Next;
        if (s != NULL) printf(",");
    }
    printf("],");
    printf("\n");

    printf("\"data\":{");
    printf("\n");

    s = NETWORK.StationList;            // Go through the stations. This is actually unnecessary since there is only one station in a dump file.
    while (s != NULL) {
        printf("\"%s\":{", s->StationID);
        printf("\n");

        printf("\"components\":[");
        for (i=0;i<strlen(ComponentStr);i++) {
            if (ComponentStr[i] != 'A')
                printf("\"%c\"", ComponentStr[i]);
            else
                printf("\"%c\"", 's');
            if (i != strlen(ComponentStr) - 1) printf(",");
        }
        printf("],");
        printf("\n");

        printf("\"resolution\":%d,", Resolution);
        printf("\n");

        printf("\"Longitude\":%.2f,", s->Longitude/100.0);
        printf("\n");

        printf("\"Latitude\":%.2f,", s->Latitude/100.0);
        printf("\n");

        FindDataTimes(s, &DataStartTime, &DataEndTime);
        SecsToStrC(DataStartTime,TimeStr);
        printf("\"data_starttime\":\"%s\",", TimeStr);
        printf("\n");
        SecsToStrC(DataEndTime,TimeStr);
        printf("\"data_endtime\":\"%s\",", TimeStr);
        printf("\n");

        // Parameter lines
        printf("\"paramlines\":[");
        printf("\n");

        for(i=0;i<ParamLineCount; i++) {
            printf("{");
            SecsToStrC(ParamLineTime[i],TimeStr);
            printf("\"time\":\"%s\",", TimeStr);
            strcpy(InfoLineStr, ParamLine[i]);
            InfoLineStr[GetLineLength(ParamLine[i])] = '\0';        // removes possible LF, CR from the string
            printf("\"line\":\"%s\"", InfoLineStr);
            printf("}");
            if (i < ParamLineCount - 1) printf(",");
            printf("\n");
        }
        printf("],");
        printf("\n");


        for (i=0;i<strlen(ComponentStr);i++) {
            c = ComponentStr[i];
            cChar = (c == 'A') ? 's' : c;

            base = ComputeAverage(s, c, StartTime+3600*StartHour, 3600*(EndHour-StartHour));
            if (base == MissingValue) {
                base = ComputeAverage(s, c, StartTime, EndTime-StartTime);  // Try the whole data period
            }
            if (base != MissingValue)
                printf("\"base%c\":%.2f,", cChar, base/100.0);
            else
                printf("\"base%c\":null,", cChar);

            printf("\"%c\":[", cChar);

            t = StartTime;
            while (t < EndTime) {
                Ave = ComputeAverage(s, c, t, AvePeriod);
                if (Ave != MissingValue)
                    printf("%.2f", (Ave-base)/100.0);
                else
                    printf("null");
                t += AvePeriod;
                if (t < EndTime) printf(",");
            }

            printf("]");
            if (i != strlen(ComponentStr) - 1) printf(",");
            printf("\n");
        }
        printf("}");
        s = s->Next;
        if (s != NULL) printf(",");
        printf("\n");
    }
    printf("}");    // End of "data"
    printf("}");    // End of json
    printf("\n");

    FreeNetwork(&NETWORK);
    return OK;
}