User Guide

class Measure  {
    String name;
    long timestamp;
    double value;
    float quality;
    Map<String, String> tags;
}

class Chunk  {
    SchemaVersion version = SchemaVersion.VERSION_1;
    String name, id, metricKey;
    byte[] value;
    long start, end;
    Map<String, String> tags;

    long count, first, min, max, sum, avg, last, stdDev;
    float qualityFirst, qualityMin, qualityMax, qualitySum, qualityAvg;

    int year, month;
    String day;
    String origin;
    String sax;
    boolean trend;
    boolean outlier;
}

<schema name="default-config" version="1.6">
    ...
  <field name="chunk_avg" type="pdouble"/>
  <field name="chunk_count" type="pint"/>
  <field name="chunk_day" type="string"/>
  <field name="chunk_end" type="plong"/>
  <field name="chunk_first" type="pdouble"/>
  <field name="chunk_hour" type="pint"/>
  <field name="chunk_last" type="pdouble"/>
  <field name="chunk_max" type="pdouble"/>
  <field name="chunk_min" type="pdouble"/>
  <field name="chunk_month" type="pint"/>
  <field name="chunk_origin" type="string"/>
  <field name="chunk_outlier" type="boolean"/>
  <field name="chunk_quality_avg" type="pfloat"/>
  <field name="chunk_quality_first" type="pfloat"/>
  <field name="chunk_quality_max" type="pfloat"/>
  <field name="chunk_quality_min" type="pfloat"/>
  <field name="chunk_quality_sum" type="pfloat"/>
  <field name="chunk_sax" type="ngramtext"/>
  <field name="chunk_start" type="plong"/>
  <field name="chunk_std_dev" type="pdouble"/>
  <field name="chunk_sum" type="pdouble"/>
  <field name="chunk_trend" type="boolean"/>
  <field name="chunk_value" type="text_general" multiValued="false" indexed="false"/>
  <field name="chunk_year" type="pint"/>
  <field name="metric_id" type="string" docValues="true" multiValued="false" indexed="true" stored="true"/>
  <field name="metric_key" type="string"/>
  <field name="name" type="string" multiValued="false" indexed="true" required="true" stored="true"/>
</schema>

cd $SOLR_HOME
# start a Solr core locally as well as a standalone zookeeper server.
bin/solr start -cloud -s $HDH_HOME/data/solr/node1 -p 8983
# start a second Solr core locally which will use the zookeeper server previously created.
bin/solr start -cloud -s $HDH_HOME/data/solr/node2 -p 7574 -z localhost: 9983

$HDH_HOME/historian-{hdh_version}/bin/historian-server.sh start

$HDH_HOME/historian-{hdh_version}/bin/ historian-server.sh stop

$HDH_HOME/grafana-{grafana_version}/bin/grafana-server start

curl --location --request POST 'http://localhost:8080/api/grafana/query' \
--header 'Content-Type: application/json' \
--data-raw '{
  "panelId": 1,
  "range": {
    "from": "2019-03-01T00:00:00.000Z",
    "to": "2020-03-01T23:59:59.000Z"
  },
  "interval": "30s",
  "intervalMs": 30000,
  "targets": [
    {
      "target": "\"ack\"",
      "type": "timeserie"
    }
  ],
  "format": "json",
  "maxDataPoints": 550
}'

curl --location --request POST 'http://localhost:8080/api/grafana/search' \
--header 'Content-Type: application/json' \
--data-raw '{
    "target": "*"
}'

curl --location --request POST 'http://localhost:8080/api/grafana/query' \
--header 'Content-Type: application/json' \
--data-raw '{
         "range": {
          "from": "2019-11-25T23:59:59.999Z",
          "to": "2019-11-30T23:59:59.999Z"
      },
      "targets": [
        { "target": "ack" }
      ]
}'

$HDH_HOME/spark-{spark-version}-bin-hadoop{hadoop_version}/bin/spark-shell --jars assembly/target/historian-{hdh_version}/historian-{hdh_version}/lib/loader-{hdh_version}.jar,assembly/target/historian-{hdh_version}/historian-{hdh_version}/lib/spark-solr-3.6.6-shaded.jar

import com.hurence.historian.model.ChunkRecordV0
import com.hurence.historian.spark.ml.Chunkyfier
import com.hurence.historian.spark.sql
import com.hurence.historian.spark.sql.functions._
import com.hurence.historian.spark.sql.reader.{MeasuresReaderType, ReaderFactory}
import com.hurence.historian.spark.sql.writer.{WriterFactory, WriterType}
import com.lucidworks.spark.util.SolrSupport
import org.apache.commons.cli.{DefaultParser, Option, Options}
import org.apache.spark.sql.SparkSession
import org.slf4j.LoggerFactory

 val filePath = "/Users/tom/Documents/workspace/historian/loader/src/test/resources/it-data-4metrics.csv.gz"
 val reader = ReaderFactory.getMeasuresReader(MeasuresReaderType.GENERIC_CSV)
    val measuresDS = reader.read(sql.Options(
      filePath,
      Map(
        "inferSchema" -> "true",
        "delimiter" -> ",",
        "header" -> "true",
        "nameField" -> "metric_name",
        "timestampField" -> "timestamp",
        "timestampDateFormat" -> "s",
        "valueField" -> "value",
        "tagsFields" -> "metric_id,warn,crit"
      )))



val chunkyfier = new Chunkyfier().setGroupByCols(Array(  "name", "tags.metric_id"))
val chunksDS = chunkyfier.transform(measuresDS).as[ChunkRecordV0]


val writer = WriterFactory.getChunksWriter(WriterType.SOLR)
    writer.write(sql.Options("historian", Map(
      "zkhost" -> "localhost:9983",
      "collection" -> "historian",
      "tag_names" -> "metric_id,warn,crit"
    )), chunksDS)